diff --git a/.gitignore b/.gitignore index 617e5c62f91..f7cc3e54b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,8 @@ bench/trace-schemas/messages/ # Test artifacts cardano-tracer/cardano-tracer-test +/cardano-tracer-test/ +/kernel-resource-summary.json # IntellIJ project folder .idea/ diff --git a/cardano-node.code-workspace b/cardano-node.code-workspace new file mode 100644 index 00000000000..2f9eb356abb --- /dev/null +++ b/cardano-node.code-workspace @@ -0,0 +1,10 @@ +{ + "folders": [ + { + "path": "." + }, + { + "path": "../hermod-tracing" + } + ] +} \ No newline at end of file diff --git a/cardano-node/cardano-node.cabal b/cardano-node/cardano-node.cabal index f50385340cc..b37da642ae4 100644 --- a/cardano-node/cardano-node.cabal +++ b/cardano-node/cardano-node.cabal @@ -106,6 +106,7 @@ library Cardano.Node.Tracing.NodeInfo Cardano.Node.Tracing.NodeStartupInfo Cardano.Node.Tracing.Render + Cardano.Node.Tracing.Span Cardano.Node.Tracing.StateRep Cardano.Node.Tracing.Tracers Cardano.Node.Tracing.Tracers.BlockReplayProgress diff --git a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs index 72aac85c5cd..cff4da15318 100644 --- a/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs +++ b/cardano-node/src/Cardano/Node/Tracing/DefaultTraceConfig.hs @@ -26,6 +26,8 @@ defaultCardanoConfig = emptyTraceConfig { [ ConfSeverity (SeverityF Nothing)]) ,(["ChainDB"], [ ConfSeverity (SeverityF (Just Info))]) + ,(["Span"], + [ ConfSeverity (SeverityF (Just Info))]) ,(["ChainDB", "AddBlockEvent", "AddBlockValidation"], [ ConfSeverity (SeverityF Nothing)]) ,(["ChainSync", "Client"], diff --git a/cardano-node/src/Cardano/Node/Tracing/Span.hs b/cardano-node/src/Cardano/Node/Tracing/Span.hs new file mode 100644 index 00000000000..e43a18914d6 --- /dev/null +++ b/cardano-node/src/Cardano/Node/Tracing/Span.hs @@ -0,0 +1,140 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | OpenTelemetry-style spans on top of the @trace-dispatcher@ framework. +-- +-- A span is a pair of correlated trace messages — 'SpanBegin' and 'SpanEnd' — +-- sharing a single 'SpanId'. 'withSpan' brackets an action so that the end +-- message is always emitted, even on exception, which is what lets a +-- Loki-style alerting system fire on \"span that never ended\". +-- +-- The duration is measured /client-side/ (in this process, monotonic clock) +-- and shipped inside 'SpanEnd', so the consumer (cardano-tracer / timeseries / +-- Prometheus) stays stateless — it never has to pair begin\/end itself. +-- +-- * For __timeseries\/Prometheus__: 'asMetrics' emits @spanDurationMs@. +-- * For __Loki (LogQL)__: 'forMachine' emits a flat JSON object with a +-- stable @span_id@ (log field, high cardinality) and @name@ +-- (label-friendly, low cardinality) plus @event=begin|end@. +module Cardano.Node.Tracing.Span + ( SpanId (..) + , SpanTrace (..) + , newSpanId + , withSpan + ) where + +import Cardano.Logging + +import Control.Exception.Safe (MonadMask, finally) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.Aeson (Value (String), (.=)) +import Data.Text (Text) +import Data.Unique (hashUnique, newUnique) +import Data.Word (Word64) +import GHC.Clock (getMonotonicTimeNSec) + +-- | Correlation id shared by a 'SpanBegin' / 'SpanEnd' pair. +-- Process-unique for the lifetime of the run (see 'newSpanId'). +newtype SpanId = SpanId { unSpanId :: Word64 } + deriving (Eq, Ord, Show) + +-- | The two ends of a span. +data SpanTrace + = SpanBegin !SpanId !Text + -- ^ Start of a span: id + human name of the operation. + | SpanEnd !SpanId !Text !Double + -- ^ End of a span: id, name, and measured duration in milliseconds. + deriving (Show) + +-- | Allocate a fresh, process-unique span id. +-- +-- Uses 'Data.Unique' so it needs no extra dependency and never blocks. Ids +-- are unique within a single node run; they are /not/ stable across restarts +-- (fine for correlating one begin with one end — which is all we need). +newSpanId :: MonadIO m => m SpanId +newSpanId = liftIO (SpanId . fromIntegral . hashUnique <$> newUnique) + +-- | Run @action@ inside a span, emitting 'SpanBegin' before and 'SpanEnd' +-- after — even if @action@ throws. +-- +-- @ +-- withSpan tr \"replayLedger\" $ do +-- ...work... +-- @ +withSpan + :: (MonadIO m, MonadMask m) + => Trace m SpanTrace -- ^ where to emit the span messages + -> Text -- ^ human name of the operation + -> m a -- ^ the work to measure + -> m a +withSpan tr name action = do + sid <- newSpanId + !t0 <- liftIO getMonotonicTimeNSec + traceWith tr (SpanBegin sid name) + action `finally` do + !t1 <- liftIO getMonotonicTimeNSec + let !ms = fromIntegral (t1 - t0) / 1e6 :: Double + traceWith tr (SpanEnd sid name ms) + +-------------------------------------------------------------------------------- +-- Formatting +-------------------------------------------------------------------------------- + +instance LogFormatting SpanTrace where + forMachine _ (SpanBegin sid name) = + mconcat + [ "kind" .= String "SpanBegin" + , "event" .= String "begin" + , "span_id" .= unSpanId sid + , "name" .= name + ] + forMachine _ (SpanEnd sid name ms) = + mconcat + [ "kind" .= String "SpanEnd" + , "event" .= String "end" + , "span_id" .= unSpanId sid + , "name" .= name + , "duration_ms" .= ms + ] + + forHuman (SpanBegin sid name) = + "Span begin [" <> showT (unSpanId sid) <> "] " <> name + forHuman (SpanEnd sid name ms) = + "Span end [" <> showT (unSpanId sid) <> "] " <> name + <> " (" <> showT ms <> " ms)" + + -- Only the end carries a measurement. The metric name embeds the span name so + -- distinct operations are distinguishable; keep the set of names SMALL to + -- avoid Prometheus/timeseries cardinality blow-up. + asMetrics (SpanBegin _ _) = [] + asMetrics (SpanEnd _ name ms) = + [ DoubleM ("spanDurationMs." <> name) ms ] + +-------------------------------------------------------------------------------- +-- Documentation / metadata +-------------------------------------------------------------------------------- + +instance MetaTrace SpanTrace where + namespaceFor SpanBegin{} = Namespace [] ["Span", "Begin"] + namespaceFor SpanEnd{} = Namespace [] ["Span", "End"] + + severityFor (Namespace _ ["Span", "Begin"]) _ = Just Info + severityFor (Namespace _ ["Span", "End"]) _ = Just Info + severityFor _ _ = Nothing + + documentFor (Namespace _ ["Span", "Begin"]) = Just + "Start of a correlated span. Carries a span_id shared with the matching \ + \Span.End, and the human name of the operation." + documentFor (Namespace _ ["Span", "End"]) = Just + "End of a correlated span. Carries the same span_id as Span.Begin plus \ + \the client-side measured duration in milliseconds." + documentFor _ = Nothing + + metricsDocFor (Namespace _ ["Span", "End"]) = + [("spanDurationMs", "Client-side measured span duration, in milliseconds")] + metricsDocFor _ = [] + + allNamespaces = + [ Namespace [] ["Span", "Begin"] + , Namespace [] ["Span", "End"] + ] diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs index 4c127580a28..dfb4fe026cc 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers.hs @@ -41,6 +41,7 @@ import Cardano.Node.Tracing.Tracers.NodeVersion (getNodeVersion) import Cardano.Node.Tracing.Tracers.Rpc () import Cardano.Node.Tracing.Tracers.Shutdown () import Cardano.Node.Tracing.Tracers.Startup () +import Cardano.Node.Tracing.Span (withSpan) import Ouroboros.Consensus.Ledger.Inspect (LedgerEvent) import Ouroboros.Consensus.MiniProtocol.ChainSync.Client (TraceChainSyncClientEvent) import qualified Ouroboros.Consensus.Network.NodeToClient as NodeToClient @@ -159,6 +160,12 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig = do !rpcTr <- mkCardanoTracer trBase trForward mbTrEKG ["RPC"] configureTracers configReflection trConfig [rpcTr] + -- Span tracer: emits correlated Span.Begin / Span.End pairs. Registered + -- through the standard pipeline so spans are forwarded to cardano-tracer + -- (and thus visible to Loki) and exposed on Prometheus/EKG as metrics. + !spanTr <- mkCardanoTracer trBase trForward mbTrEKG [] + configureTracers configReflection trConfig [spanTr] + traceTracerInfo trBase trForward configReflection let warnings = checkNodeTraceConfiguration' trConfig @@ -169,6 +176,11 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig = do traceWith nodeVersionTr getNodeVersion + -- Demonstration span emitted once at tracer initialisation. This proves the + -- begin/end pair flows through the pipeline; replace with real withSpan + -- calls around the operations you want to measure. + withSpan spanTr "nodeTracersInit" (pure ()) + pure Tracers { chainDBTracer = mkTracer (traceWith chainDBTr') diff --git a/cardano-tracer/bench/cardano-tracer-bench.hs b/cardano-tracer/bench/cardano-tracer-bench.hs index 388db5dfd03..a6d8e291983 100644 --- a/cardano-tracer/bench/cardano-tracer-bench.hs +++ b/cardano-tracer/bench/cardano-tracer-bench.hs @@ -61,6 +61,7 @@ main = do , teStateDir = Nothing , teMetricsHelp = [] , teTimeseriesHandle = Nothing + , teAlarmRegistry = Nothing } removePathForcibly root @@ -118,6 +119,7 @@ main = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams root FileMode format] , rotation = Nothing diff --git a/cardano-tracer/cardano-tracer.cabal b/cardano-tracer/cardano-tracer.cabal index 50cd5f2f83c..75a535da31c 100644 --- a/cardano-tracer/cardano-tracer.cabal +++ b/cardano-tracer/cardano-tracer.cabal @@ -61,6 +61,15 @@ library Cardano.Tracer.Acceptors.Server Cardano.Tracer.Acceptors.Utils + Cardano.Tracer.Handlers.Alarms.Auth + Cardano.Tracer.Handlers.Alarms.Consumers + Cardano.Tracer.Handlers.Alarms.Registry + Cardano.Tracer.Handlers.Alarms.Server + Cardano.Tracer.Handlers.Alarms.Store + Cardano.Tracer.Handlers.Alarms.TimeseriesRules + Cardano.Tracer.Handlers.Alarms.TraceRules + Cardano.Tracer.Handlers.Alarms.Types + Cardano.Tracer.Handlers.Logs.File Cardano.Tracer.Handlers.Logs.Journal Cardano.Tracer.Handlers.Logs.Rotator @@ -276,7 +285,9 @@ test-suite cardano-tracer-test main-is: cardano-tracer-test.hs - other-modules: Cardano.Tracer.Test.Forwarder + other-modules: Cardano.Tracer.Test.Alarms.Tests + Cardano.Tracer.Test.Alarms.TimeseriesTests + Cardano.Tracer.Test.Forwarder Cardano.Tracer.Test.DataPoint.Tests Cardano.Tracer.Test.Logs.Tests Cardano.Tracer.Test.Restart.Tests @@ -286,6 +297,7 @@ test-suite cardano-tracer-test build-depends: aeson , async , bytestring + , cardano-timeseries-io , cardano-tracer , cborg , containers diff --git a/cardano-tracer/docs/alarm-system-concept.md b/cardano-tracer/docs/alarm-system-concept.md new file mode 100644 index 00000000000..b8186a1df11 --- /dev/null +++ b/cardano-tracer/docs/alarm-system-concept.md @@ -0,0 +1,493 @@ +# Alarm system concept for `cardano-tracer` + +Status: proposal + +## Summary + +`cardano-tracer` should provide a central alarm service for alarms detected by: + +1. `hermod-recon`, running as a separate process; +2. rules evaluated against the timeseries store in `cardano-tracer`; and +3. trace messages received by `cardano-tracer` whose severity is at or above a + configured threshold. + +Both sources publish the same immutable `AlarmEvent` representation. The alarm +service stores accepted events, fans them out to statically configured consumers, +and makes them available to authorised clients through a REST API. + +An alarm is a one-shot event. There is no acknowledgement, ownership, or +active/resolved lifecycle in this proposal. + +## Goals + +- Give Hermod and timeseries rules one consistent way to raise alarms. +- Decouple alarm detection from alarm delivery. +- Make alarm consumers and their filters configurable. +- Preserve enough evidence to understand why an alarm was raised. +- Prevent a continuously true timeseries rule or a flood of repeated trace + messages from producing an event on every occurrence. +- Keep the set of delivery mechanisms open until the initial consumers are + selected. + +## Non-goals + +- Incident management, acknowledgement, assignment, or resolution. +- Running `hermod-recon` inside the `cardano-tracer` process. +- Defining a new rule language. Hermod continues to use its LTL language and + timeseries rules use the existing `cardano-timeseries-io` query language. +- Letting remote clients create arbitrary timeseries rules through the API. + +## Terminology + +- **Producer**: a component that detects a condition and submits an alarm. +- **Rule**: configured detection logic, identified by a stable `ruleId`. +- **Alarm event**: the immutable record produced by a rule occurrence. +- **Consumer**: a statically configured destination to which the service pushes + matching alarm events. + +## Architecture + +```text + node trace files ---> hermod-recon ---- HTTP POST ----+ + | + node metrics ---> timeseries store ---> rule engine +--> alarm service + | | + node trace messages ---> trace severity rules ------+ +--> alarm store + | +--> configured consumers + | `--> REST history + | + `--> common AlarmEvent +``` + +The alarm service is a component of `cardano-tracer`, but should be kept separate +from the existing legacy email-notification code. The latter groups trace messages +by severity and has a different data model. Its email implementation may later be +reused by an email alarm consumer. + +### Internal components + +1. **Ingress API** validates, authenticates, normalises, and deduplicates alarms + submitted by external producers such as Hermod. +2. **Timeseries rule evaluator** periodically executes configured queries using + the existing in-process `TimeseriesHandle`. +3. **Trace severity matcher** checks every trace message received from a node + against the configured trace severity rules. +4. **Alarm store** assigns an event ID and retains accepted events for history. +5. **Dispatcher** matches accepted events against configured consumers and hands + them to consumer-specific workers. +6. **History API** serves filtered history to authorised clients. + +An event is visible to consumers and readers only after it has been accepted +by the alarm store. + +## Alarm event + +All producers are normalised to a versioned envelope. A representative JSON +event is: + +```json +{ + "schemaVersion": 1, + "eventId": "0198b80a-cad7-7b2d-95cb-37ee0dd3ee81", + "sourceEventId": "hermod-mainnet-42-1731529958123", + "raisedAt": "2026-07-13T12:34:56.123Z", + "receivedAt": "2026-07-13T12:34:56.184Z", + "source": "hermod-recon", + "ruleId": "chain-growth-42", + "severity": "critical", + "summary": "Chain did not grow within the expected interval", + "scope": { + "network": "mainnet", + "nodeId": "relay-1" + }, + "labels": { + "team": "node-operations", + "site": "eu-central" + }, + "details": { + "formulaIndex": 42, + "formula": "...", + "relevance": [] + } +} +``` + +Required producer fields are `sourceEventId`, `raisedAt`, `ruleId`, `severity`, +and `summary`. `cardano-tracer` supplies `eventId`, `receivedAt`, and the trusted +`source` associated with the producer credential. It must not trust a caller to +choose its own source identity. + +`severity` should use the existing Cardano severity vocabulary: +`debug`, `info`, `notice`, `warning`, `error`, `critical`, `alert`, and +`emergency`. Alarm configurations should normally use `warning` or higher. + +`scope` contains well-known routing dimensions. `labels` is an extensible map for +consumer filtering. `details` is source-specific JSON and must have configured +size and nesting limits. + +The pair `(source, sourceEventId)` is the idempotency key. Re-submitting the same +event returns the already assigned `eventId` and does not dispatch it twice. + +## Producers + +### Hermod ReCon + +`hermod-recon` remains a separate process. On a `FormulaNegativeOutcome`, it +submits one alarm to the ingress API. The mapping is: + +| Hermod value | Alarm field | +| --- | --- | +| configured rule name, or formula index as fallback | `ruleId` | +| formula index | `details.formulaIndex` | +| formula | `details.formula` | +| relevant trace events | `details.relevance` | +| configured rule severity | `severity` | +| configured rule summary | `summary` | + +Hermod should use a structured alarm output/sink. Human-readable output and the +current `--grep` output are not a stable integration contract. As an incremental +migration, a small adapter process may translate Hermod's machine-formatted +`FormulaNegativeOutcome` trace into the ingress request, but the preferred result +is a native Hermod HTTP alarm sink. + +The Hermod worker owns retry with exponential backoff for transient failures. +It uses the same `sourceEventId` on every retry so retries are safe. A permanent +validation or authorisation error is logged locally and is not retried forever. + +### Timeseries rules + +Timeseries rules are configured in `cardano-tracer` and evaluated in process. +A rule consists of: + +- a stable identifier, summary, severity, labels, and optional node scope; +- an existing timeseries query that evaluates to a boolean vector; +- an evaluation interval; +- an optional `for` duration for which the expression must remain true; and +- an optional repeat interval. + +For every output series, the evaluator maintains a small state machine: + +```text +false/missing -- true --> pending -- true for configured duration --> publish + ^ | | + `------ false/missing --'---- false/missing <------------------' +``` + +The default is edge-triggered: one event is published when the rule first meets +its `for` duration. It does not publish a resolution event. After the expression +becomes false or missing, a later false-to-true transition may publish a new +alarm. When `repeatEvery` is configured, a still-true rule may publish reminders +at that interval. + +Each output series must generate a distinct deterministic series key from its +labels. The series key is included in `sourceEventId` and in the alarm labels. +Evaluation errors and missing data are traced as health information, not silently +interpreted as alarms. A future rule option may explicitly define missing data as +alarming. + +The evaluator must put limits on query execution time, result cardinality, and +concurrent evaluations so alarm rules cannot starve normal timeseries ingestion +and queries. + +### Trace severity rules + +`cardano-tracer` already receives every trace message forwarded by the connected +nodes. A trace severity rule raises an alarm when a received message's severity +is at or above a configured threshold. A rule consists of: + +- a stable `ruleId` and an optional summary; +- a `threshold` severity (using the same lowercase severity vocabulary); and +- an optional `suppressForSecs` window (default 300 seconds) and optional labels. + +The mapping from a matching trace message to the alarm event is: + +| Trace message value | Alarm field | +| --- | --- | +| message severity | `severity` | +| message timestamp | `raisedAt` | +| node name | `scope.nodeId` | +| namespace (dot-joined) | `labels.namespace` | +| machine-readable message, hostname, thread ID | `details` | + +The trusted `source` is the constant `trace`: like the future timeseries rules, +this is an internal producer that never passes HTTP authentication, so its +identity is fixed rather than derived from a credential. + +Flood prevention reuses the idempotency key instead of a separate rate limiter: +the `sourceEventId` is `ruleId:node:namespace:windowIndex`, where the window +index is the message timestamp divided by `suppressForSecs`. All matches from +the same node and namespace within one window therefore share the +`(source, sourceEventId)` key and collapse into a single alarm; a match in the +next window raises a fresh one. + +## Consumer model + +A consumer implementation has the following conceptual interface: + +```text +initialise(config) -> worker +matches(filter, AlarmEvent) -> Bool +deliver(worker, AlarmEvent) -> delivered | retryable-error | permanent-error +shutdown(worker) +``` + +Each configured consumer has: + +- a unique name and consumer type; +- type-specific destination settings and credentials; +- a filter; +- queue capacity, retry policy, and timeout; and +- an enabled flag. + +Filters can select `source`, `ruleId`, minimum severity, scope fields, and labels. +The first implementation should support conjunction only. More complex boolean +filter expressions can be added later without changing `AlarmEvent`. + +Consumer types are intentionally not fixed by this concept. Likely initial +implementations are `webhook`, `email`, and `log`. A webhook is a useful generic +first delivery mechanism because downstream systems can adapt it to chat or +incident-management products. + +Consumer workers use bounded, independent queues. A slow or broken destination +must not block alarm ingestion or other consumers. Delivery is at least once for +consumers that enable retries; a destination should therefore deduplicate using +`eventId`. Exhausted delivery attempts are retained as operational failures and +traced by `cardano-tracer`. They do not create alarm events recursively. + +## API + +The alarm API should use a separately configurable endpoint, even if its server +implementation later shares code with the timeseries server. + +### Producer ingress + +```text +POST /alarms/v1/events +Authorization: Bearer +Content-Type: application/json +``` + +- `201 Created` for a newly accepted event; +- `200 OK` for an idempotent replay; +- `400 Bad Request` for invalid input; +- `401 Unauthorized` or `403 Forbidden` for rejected credentials; and +- `429 Too Many Requests` when producer limits are exceeded. + +The response contains `eventId`, `receivedAt`, and whether the event was newly +created. + +### History + +```text +GET /alarms/v1/events?after=&limit=100&source=hermod-recon&minSeverity=warning +Authorization: Bearer +``` + +Results are ordered by the store cursor and use cursor pagination. Supported +filters mirror the simple consumer filters. Retention is configurable. + +## Configuration sketch + +The precise Haskell representation can be refined during implementation. This +YAML illustrates the intended operator-facing configuration: + +```yaml +alarms: + endpoint: + epHost: "127.0.0.1" + epPort: 3210 + epForceSSL: true + retention: + maxAge: 7d + maxEvents: 100000 + limits: + maxEventBytes: 262144 + ingressQueue: 1000 + authentication: + producers: + - name: hermod-mainnet + tokenFile: "/run/secrets/hermod-alarm-token" + source: hermod-recon + readers: + - name: operations-dashboard + tokenFile: "/run/secrets/dashboard-alarm-token" + allowHistory: true + filter: + minSeverity: warning + labels: + team: node-operations + timeseriesRules: + - ruleId: high-ping-latency + summary: "Average node ping latency is above 500 ms" + severity: warning + query: "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + evaluateEvery: 30s + for: 2m + labels: + team: node-operations + traceRules: + - ruleId: error-traces + summary: "A trace message with severity error or above was received" + threshold: error + suppressForSecs: 300 + labels: + team: node-operations + consumers: + - name: alarm-audit-log + type: log + enabled: true + filter: + minSeverity: warning +``` + +Secrets should be read from protected files or an equivalent secret provider, +not written directly into the main configuration. TLS uses the existing +`tlsCertificate` configuration initially. Deployments that terminate TLS at a +trusted reverse proxy must restrict direct access to the alarm listener. + +## Storage and delivery guarantees + +The store is append-only from the alarm domain's perspective because alarm events +are immutable. It must support: + +- atomic insert with idempotency-key uniqueness; +- cursor-ordered reads and filtered history; +- retention by age and maximum count; and +- recovery across `cardano-tracer` restarts. + +The concrete backend should sit behind a small `AlarmStore` interface. An embedded +transactional database is preferable to ad-hoc JSON files once retry state and +cursor pagination are implemented. Storage failure rejects ingress rather than +publishing an event that cannot subsequently be replayed. + +The core guarantee is: after the service reports an event as accepted, the event +is durable until retention removes it. Configured consumers receive at-least-once +delivery when retries are enabled. + +## Security + +- Deny producer and history access by default. +- Give producers, readers, and administrators distinct credentials and roles. +- Derive the producer source from credentials. +- Apply per-credential request and connection limits. +- Bound event size, label count, label lengths, JSON depth, and query result + cardinality. +- Do not include credentials in traces or alarm details. +- Treat Hermod relevance evidence as potentially sensitive operational data; + consumer and reader filters are an authorisation boundary. +- Refuse to start an externally reachable clear-text endpoint unless an explicit + insecure setting is enabled. + +The first version does not require a remote administration API. Rules, consumers, +and credentials are managed through configuration and take effect on restart. +Hot reload can be considered separately. + +## Observability + +`cardano-tracer` should expose metrics and structured traces for: + +- accepted, rejected, duplicate, and rate-limited ingress events; +- timeseries rule evaluations, evaluation failures, and evaluation duration; +- alarms published by source, rule, and severity, with bounded label cardinality; +- per-consumer queue depth, attempts, successes, retries, permanent failures, + and drops; and +- alarm-store size, retention removals, and failures. + +Alarm infrastructure failures must not create alarms through the same pipeline, +which could cause a feedback loop. They are emitted as normal internal traces and +metrics. + +## Failure behaviour + +- If Hermod cannot reach `cardano-tracer`, Hermod retries with its stable + idempotency key and bounded local buffering. +- If the store is unavailable, external ingress fails and timeseries alarm + publication is retried without advancing the rule's published state. +- If one consumer is unavailable, its worker retries independently while other + consumers continue. +- If a timeseries query fails, the rule reports an evaluation failure and keeps + its previous state until a configured stale-state timeout; it does not publish + a false alarm. +- If `cardano-tracer` restarts, persisted rule state avoids duplicate alarm edges. + When state is unavailable, the evaluator observes one complete `for` duration + before publishing. + +## Suggested implementation phases + +### Phase 1: alarm core and Hermod ingress + +- Introduce `AlarmEvent`, validation, store, and idempotent ingress. +- Add the authenticated history endpoint. +- Add a `log` consumer to exercise filtering and dispatch. +- Add either the native Hermod HTTP sink or the temporary machine-trace adapter. + +### Phase 2: timeseries evaluation + +- Add configuration and validation for timeseries rules. +- Implement edge/`for` state and persistence. +- Add evaluator safety limits and metrics. + +### Phase 3: production consumers + +- Select and implement the required consumer types. +- Add independent retry queues and persisted delivery attempts where required. +- Add operational dashboards and retention tooling. + +## Possible future features + +The features in this section are not part of the current design. They are +recorded here so a later revision can pick them up without redesigning the +core. + +### Live subscription (SSE) + +A **subscriber** is an authorised API client that receives a live stream of +alarm events. Subscribers are deliberately different from consumers: consumers +are controlled by the operator's configuration and can implement reliable +delivery, while subscribers are dynamic clients and receive a best-effort live +stream. Guaranteeing delivery to a live client while it is disconnected is +explicitly out of scope. + +```text +GET /alarms/v1/stream?source=timeseries&minSeverity=warning +Authorization: Bearer +Accept: text/event-stream +Last-Event-ID: +``` + +Server-Sent Events (SSE) is recommended over WebSockets because the data flow +is one-way, it works with ordinary HTTP infrastructure, and it has a standard +reconnection model. Events use `event: alarm`, an event ID/cursor, and the JSON +`AlarmEvent` as data. Periodic comments act as heartbeats. + +Subscription is disabled unless `allowSubscriptions` is true. A reader +credential gains an `allowSubscription` flag; its allowed filter acts as a +ceiling that a requested filter can only narrow, never broaden. If retained +history contains the `Last-Event-ID`, the server replays later events before +switching to live delivery. Otherwise it reports that the cursor is outside +retention and the client must resume from the oldest available cursor. The +alarm store's cursor doubles as the SSE resumption cursor, so history and +stream resumption share the same pagination mechanism. + +Each subscriber gets a bounded queue (a `subscriberQueue` size under `limits` +in the configuration). SSE delivery is best effort: if a client is slow and +its queue fills, only that client is disconnected, and it reconnects using its +last processed event ID. Subscription access is denied by default, and the SSE +queue size joins the list of bounded resources. + +Observability for this feature adds connected SSE subscribers and slow-client +disconnects to the exposed metrics and traces. + +## Decisions still open + +The following choices do not block the architecture: + +1. Which push consumers are required first (`webhook`, `email`, a message broker, + or another integration). +2. Which embedded storage backend should implement `AlarmStore`. +3. Whether Hermod Recon gains a native HTTP alarm sink immediately or initially uses an + adapter. +4. Whether configuration hot reload is needed. +5. Whether consumer retry queues must survive process restart in the first + release. + diff --git a/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md new file mode 100644 index 00000000000..51042aec841 --- /dev/null +++ b/cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md @@ -0,0 +1,293 @@ +# Porting the cardano-parts Grafana alerts to timeseries alarm rules + +Status: guide (companion to `alarm-system-concept.md`) + +The SRE alert catalogue in +[cardano-parts](https://github.com/input-output-hk/cardano-parts/tree/main/templates/cardano-parts-project/flake/opentofu/grafana/alerts) +defines Grafana-managed Prometheus alert rules for Cardano deployments. This +document shows how the `cardano-node` alerts from that catalogue map onto +timeseries rules in the new alarm system (Phase 2 of the concept doc) — what +translates directly, what changes, and what does not carry over. + +The comparison is against the four node-related files: +`cardano-node.nix-import`, `cardano-node-forge.nix-import`, +`cardano-node-network.nix-import`, `cardano-node-quality.nix-import`, and +`cardano-node-divergence.nix-import`. + +## Rule anatomy: Grafana vs. alarm system + +| Grafana / Prometheus rule | Alarm system `timeseriesRules` entry | +| --- | --- | +| `alert` (name) | `ruleId` | +| `expr` (PromQL) | `query` (`cardano-timeseries-io` language) | +| `for` | `for` | +| `labels.severity = "page"` | `severity:` — richer vocabulary; use `critical` for pages, lower levels become possible | +| `annotations.summary/description` | `summary` (static text; no `{{$labels.instance}}` templating) | +| node identity via `instance` label | series key labels (`node_name`) carried into the alarm's labels | +| evaluation interval (Grafana global) | `evaluateEvery` per rule | + +The alarm state machine matches Grafana's semantics: edge-triggered +publication after the expression has been true for the `for` duration, no +resolution events, optional `repeatEvery` reminders. + +## Query language correspondence + +The `cardano-timeseries-io` language covers almost every PromQL construct the +catalogue uses: + +| PromQL | timeseries query language | +| --- | --- | +| instant selector `m` | `m now` — a metric is a *function of time* and must be applied | +| `m[5m]` (range) | `m[now - 5m; now]` (the metric stays unapplied inside a range) | +| `m[360m:1m]` (subquery w/ step) | `m[now - 360m; now : 1m]` | +| `rate(...)`, `increase(...)` | `rate (...)`, `increase (...)` — **per millisecond**, not per second, and without counter-reset handling or window extrapolation | +| `avg_over_time`, `sum_over_time` | same names | +| `quantile_over_time(0.95, v)` | `quantile_over_time 0.95 v` | +| `quantile by(environment) (0.2, v)` | `quantile_by ("environment") 0.2 v` | +| `min`/`max`/`avg`/`abs`/`round` | same names | +| `A unless B` | `unless A B` | +| `m{environment="mainnet"}` | `(m now){"environment" = "mainnet"}` (`=` and `!=` only; label keys are quoted) | +| `expr > k` (alert firing) | plain `>`/`==`/… on an instant vector is a **filter**, exactly like PromQL alert expressions: series failing the comparison drop out of the result, and the rule evaluator treats a series' *presence* as "condition true". Scalar comparisons yield real booleans. | +| `and` / `or` (scalar logic) | `&&` / `||` | +| — | extras: `let`/lambdas, `map`, `filter`, `join`, `to_scalar`, `earliest`, `latest`, `metrics` | + +Missing relative to PromQL: regex label matchers (`=~`, `!~`), vector-`or` as +union/fallback, and cross-series binary matching (`on() group_right()`). +Workarounds below. + +Two further differences that bit during prototyping: `rate` values are per +millisecond, so PromQL thresholds on rates must be divided by 1000 (`> 0.5` +per second becomes `> 0.0005`; comparisons against `0` are unaffected), and +`rate` over a window that holds only a single sample is an evaluation error +(traced, never published as an alarm). Durations in the rule configuration +(`evaluateEvery`, `for`, `repeatEvery`) are plain integer **seconds**, not +Prometheus duration strings. + +## The catalogue, translated + +Severity suggestions replace the one-size-fits-all `page`. Metric names are +the new-tracing names as forwarded by the node — verify against a live store +with the `metrics` query before deploying (see caveats). + +Durations (`evaluateEvery`, `for`) are plain integer seconds. Instant +selectors are applied to `now`; range windows take the metric unapplied. + +```yaml +alarms: + timeseriesRules: + # --- cardano-node.nix-import ------------------------------------------- + - ruleId: blockheight-unchanged # cardano_node_blockheight_unchanged + summary: "Blockheight unchanged for more than 7 minutes" + severity: critical + query: "rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0" + evaluateEvery: 30 + for: 120 + + - ruleId: mempool-high # cardano_node_mempool_high + summary: "More than 200 transactions in mempool for over 10 minutes" + severity: warning + query: "cardano_node_metrics_txsInMempool_int now > 200" + evaluateEvery: 30 + for: 600 + + - ruleId: mempool-soft-timeouts # cardano_node_mempool_soft_timeouts_detected + summary: "Mempool soft timeouts detected in the past hour" + severity: warning + query: "increase (cardano_node_metrics_txsMempoolTimeoutSoft_counter[now - 1h; now]) > 2" + evaluateEvery: 60 + for: 60 + + - ruleId: mempool-hard-timeouts # cardano_node_mempool_hard_timeouts_detected + summary: "Mempool hard timeouts detected in the past hour" + severity: error + query: "increase (cardano_node_metrics_txsMempoolTimeoutHard_counter[now - 1h; now]) > 0" + evaluateEvery: 60 + for: 60 + + # Needs prototyping before use: the bare comparison inside the subquery + # ((m != 0)[...]) does not elaborate -- a metric must be applied to a + # time, which inside a subquery requires a lambda. Like the divergence + # family below, verify against the real interpreter first. + - ruleId: blockheight-metric-missing # cardano_node_metric_missing + summary: "Blockheight metric missing for more than 10 minutes" + severity: error + query: >- + unless (sum_over_time ((cardano_node_metrics_blockNum_int != 0)[now - 360m; now : 1m]) < 350) + cardano_node_metrics_blockNum_int + evaluateEvery: 60 + for: 60 + + # --- cardano-node-forge.nix-import ------------------------------------- + - ruleId: no-blocks-forged-24h # cardano_node_forge_blocks_missing + summary: "No blocks forged in the past 24 hours" + severity: critical + query: "increase (cardano_node_metrics_blocksForged_int[now - 24h; now]) == 0" + evaluateEvery: 300 + for: 60 + + - ruleId: forged-not-adopted # cardano_node_forge_not_adopted_error + summary: "Failed to adopt one or more forged blocks in the past hour" + severity: error + query: "increase (cardano_node_metrics_Forge_didnt_adopt_counter[now - 1h; now]) > 0" + evaluateEvery: 60 + for: 60 + + - ruleId: cannot-forge # cardano_node_cannot_forge_new_tracing + summary: "Failed to forge one or more blocks in the past hour" + severity: error + query: "increase (cardano_node_metrics_nodeCannotForge_int[now - 1h; now]) > 0" + evaluateEvery: 60 + for: 60 + + - ruleId: slot-leadership-checks-missed # too_many_slot_leadership_checks_missed + summary: "Slot leadership checks missed for more than half of slots" + severity: critical + # rate is per millisecond: PromQL's 0.5/s threshold becomes 0.0005/ms. + query: "rate (cardano_node_metrics_slotsMissed_int[now - 5m; now]) > 0.0005" + evaluateEvery: 30 + for: 120 + + # KES: the richer severity vocabulary replaces three identical "page" + # alerts with an escalation ladder. + - ruleId: kes-expiry-10-periods # cardano_node_KES_expiration_metric_10period_notice + summary: "Less than 10 KES periods remaining" + severity: warning + query: "cardano_node_metrics_remainingKESPeriods_int now <= 10" + evaluateEvery: 300 + for: 300 + - ruleId: kes-expiry-5-periods + summary: "Less than 5 KES periods remaining" + severity: error + query: "cardano_node_metrics_remainingKESPeriods_int now <= 5" + evaluateEvery: 300 + for: 300 + - ruleId: kes-expiry-1-period + summary: "KES expires within 1 period" + severity: critical + query: "cardano_node_metrics_remainingKESPeriods_int now <= 1" + evaluateEvery: 300 + for: 300 + + # --- cardano-node-network.nix-import ------------------------------------ + # This one is the concept doc's own example rule. + - ruleId: high-ping-latency # high_cardano_ping_latency + summary: "Average node ping latency above 500 ms" + severity: warning + query: "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + evaluateEvery: 30 + for: 3600 + + - ruleId: block-adoption-delay # blocks_adoption_delay_too_high + summary: "95th-percentile block adoption delay above 4.5 s" + severity: warning + query: "avg (quantile_over_time 0.95 (cardano_node_metrics_blockfetchclient_blockdelay_real[now - 6h; now])) >= 4.5" + evaluateEvery: 300 + for: 60 + + - ruleId: block-utilization-high # blocks_utilization_too_high + summary: "Average block utilization above 95%" + severity: warning + query: "100 * avg (avg_over_time (cardano_node_metrics_blockfetchclient_blocksize_int[now - 6h; now]) / 90112) > 95" + evaluateEvery: 300 + for: 300 + + - ruleId: blockfetch-delay-high # cardano_blockfetchclient_blockdelay_high + summary: "Less than 90% of blocks arriving within 5 seconds" + severity: warning + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real now < 0.90" + evaluateEvery: 60 + for: 600 + + - ruleId: blockfetch-delay-critical # cardano_blockfetchclient_blockdelay_critical + summary: "Less than 50% of blocks arriving within 5 seconds" + severity: critical + query: "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real now < 0.50" + evaluateEvery: 60 + for: 600 + + - ruleId: connection-count-high # cardano_connection_count_high + summary: "Incoming connection count above 450 (hard limit 512)" + severity: warning + query: "cardano_node_metrics_connectionManager_inboundConns_int now > 450" + evaluateEvery: 60 + for: 600 + + # --- cardano-node-quality.nix-import ------------------------------------ + # Fleet-wide rule: only meaningful when the whole fleet forwards to this + # tracer (see caveats). The regex exclusion {environment!~"preview"} has + # no equivalent; use != or per-environment rules. + - ruleId: chain-density-degraded # chain_quality_degraded + summary: "More than 20% of nodes below 70% chain density" + severity: warning + query: '100 * quantile_by ("environment") 0.2 ((cardano_node_metrics_density_real now){"environment" != "preview"} * 20) < 70' + evaluateEvery: 60 + for: 300 +``` + +## Alerts that do not translate one-to-one + +**Block divergence (`cardano-node-divergence.nix-import`).** The PromQL uses +cross-series matching (`max(m) - on() group_right() m`) to compare each node +against the fleet maximum. The language has no vector matching, but its +functional extras can express the same idea: + +```text +let peak = to_scalar (max (cardano_node_metrics_blockNum_int)) in +map (\x -> abs (peak - x) > 6) cardano_node_metrics_blockNum_int +``` + +combined with the analogous slot-lag condition via `&&`. This is the one +family where the translation needs prototyping against the real interpreter +before committing to a rule. + +**Elevated restarts (`cardano_node_elevated_restarts`).** Uses +`time() - nodeStartTime` inside a subquery. `now` and timestamp arithmetic +exist, and subqueries with a step are supported, so this is expressible in +principle — but timestamp-vs-number typing needs verification. A simpler +first version: alarm when `cardano_node_metrics_nodeStartTime_int` changed +within the window, or leave restart detection to the trace severity rules +(the node logs its startup). + +**Old/new tracing metric-name pairs.** Half the catalogue exists twice +(`...Forge_forged_int` vs `...blocksForged_int`) joined by PromQL vector +`or`, because Prometheus may scrape either naming scheme. `cardano-tracer` +receives metrics from the node's own forwarder, i.e. the new-tracing names +only — the `_new_tracing` variants are the ones to port, and the `or` +fallback disappears. + +## General caveats + +- **Scope.** Grafana alerts run against a central Prometheus that scrapes the + whole fleet. A `cardano-tracer` store contains only the nodes forwarding to + that tracer instance. Per-node rules (mempool, KES, forging) translate + directly; fleet-wide statistics (chain density, divergence) are only + meaningful when the relevant fleet shares one tracer. +- **Labels.** The store's series label is `node_name` (set by the acceptor), + not Prometheus's `instance`/`environment`. Rules that filter on + `environment` need that label to exist in the store, or need the filter + dropped. Alarm routing dimensions come from the rule's `labels`/scope + config plus the series key, not from annotation templates. +- **Metric names.** Names pass through `sanitiseMetricName` on ingestion. + Before writing a rule, list what the store actually holds: + `GET /timeseries/query?query=metrics`. +- **Delivery.** Grafana pages via its contact points; here, delivery is the + consumer configuration (today `log`; `webhook`/`email` planned). Porting + the rules is independent of porting the paging integration. +- **Testing.** Each ported rule should get a Level-1 test as described in + `timeseries-alarm-testing.md`: insert samples that straddle the threshold + and the `for` window, evaluate at fixed timestamps, assert exactly one + alarm. Three rules of this catalogue — `mempool-high`, + `blockheight-unchanged`, and `high-ping-latency` — already have such + tests (`Cardano.Tracer.Test.Alarms.TimeseriesTests`), one alarming and + one quiet dataset each; they are the template for porting the rest. + +## Status + +The Phase-2 timeseries rule evaluator from the concept doc is implemented +(`Cardano.Tracer.Handlers.Alarms.TimeseriesRules`, driven by +`runTimeseriesEvaluator`). The catalogue above is its acceptance suite — +three rules are verified by the example tests; the queries of the remaining +rules follow the same corrected conventions but should each get their +Level-1 test before being relied on. The two families flagged above +(divergence, `blockheight-metric-missing`) still need prototyping against +the interpreter. diff --git a/cardano-tracer/docs/timeseries-alarm-testing.md b/cardano-tracer/docs/timeseries-alarm-testing.md new file mode 100644 index 00000000000..ca27966f11d --- /dev/null +++ b/cardano-tracer/docs/timeseries-alarm-testing.md @@ -0,0 +1,208 @@ +# Writing test cases for timeseries alarm rules + +Status: guide (companion to `alarm-system-concept.md`) + +This document describes how to write test cases for alarms driven by +timeseries rules (Phase 2 of the alarm concept), using the pieces that exist +today: the timeseries store with its `insert`/`execute` API, the alarm +registry, and the test patterns already used in `cardano-tracer-test`. + +## The data entry point: `insert` + +Test data is added through the in-process API of `cardano-timeseries-io` +(`Cardano.Timeseries.Component`): + +```haskell +insert :: TimeseriesHandle -> SeriesIdentifier -> Timestamp -> [(MetricIdentifier, Double)] -> IO () +execute :: TimeseriesHandle -> Timestamp -> Text -> IO (Either ExecutionError Value) +``` + +- A series is identified by labels, e.g. `Set.fromList [("node_name", "node-1")]` -- + the same shape production uses when node metrics arrive + (`Cardano.Tracer.Acceptors.Utils.store`). +- Timestamps are milliseconds and are **supplied by the caller**. This is what + makes tests deterministic: insert samples at chosen times and evaluate the + query at a chosen `at`, instead of racing the real clock. + +There is deliberately **no HTTP insert endpoint** -- the timeseries server only +serves `query`, `prune`, `config`, and `nodes`. In production, data enters +only as metrics forwarded from nodes. Test strategies have to respect that. + +## Level 1 (recommended): in-process tests + +Follow the style of `Cardano.Tracer.Test.Alarms.Tests`: no server, no +processes, everything driven directly. This is the right level for testing +rule logic (thresholds, `for` durations, edge-triggering, repeat intervals). + +```haskell +propHighLatencyRaisesAlarm :: Property +propHighLatencyRaisesAlarm = once $ ioProperty do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + handle <- Timeseries.create @(Tree Double) (timeseries bundle) (Just noPruneConfig) + registry <- newAlarmRegistry (assorted bundle) testAlarmsConfig + + -- 1. Arrange: insert all samples up front (queries only look backward) + let t0 = 1_700_000_000_000 -- fixed ms timestamp + series = Set.fromList [("node_name", "node-1")] + for_ [0 .. 10] \i -> + Timeseries.insert handle series (t0 + i * 30_000) [("ping_latency_ms", 640)] + + -- 2. Act: evaluate all configured rules at fixed instants, one round + -- per simulated 30 s (the state machine needs several rounds to + -- cross the rule's `for` duration) + for_ [1 .. 10] \k -> + evaluateTimeseriesRules registry handle + (posixSecondsToUTCTime (fromIntegral (t0 `div` 1000 + 30 * k))) + + -- 3. Assert: exactly one alarm in the store + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 1) +``` + +`evaluateTimeseriesRules :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> IO ()` +(and its single-rule sibling `evaluateTimeseriesRule`) in +`Cardano.Tracer.Handlers.Alarms.Registry` **take the evaluation time +explicitly** (like `pruneOnce` and `traceRuleRequest` already do); only the +production `evaluatorLoop` supplies `getCurrentTime`. This is what makes the +rounds above deterministic. + +The same pattern also works for testing query semantics alone: insert known +samples, call `execute handle at query`, and assert on the returned `Value` +(thresholds crossing true/false, missing data, window boundaries). + +Determinism pitfalls, learned the hard way (all verified against +`cardano-timeseries-io`): + +- **Create the store with pruning disabled** (`pruningPeriodMillis = + Nothing`). The pruner compares against the real wall clock and silently + deletes fixed historical test timestamps. +- **Instant lookups have a 300 s staleness bound** — a rule evaluated at `t` + sees a series only if it has a sample in `(t - 300 s, t]`, so test data + must keep sampling at least that often across the whole horizon. +- **Keep timestamps large.** The Tree store's staleness-window arithmetic is + on `Word64` and underflows for query times below 300 000 ms. +- **`rate` has no value on a single-point series** (hard error, traced as an + evaluation failure). Start evaluation rounds only after the range window + contains at least two populated grid points. +- **A bare metric is a function of time**: instant threshold rules must + apply it, `m now > 200`; range forms `m[now - 5m; now]` take the metric + unapplied. +- **Comparisons on instant vectors are filters**, not boolean vectors: the + result keeps the surviving series with their original values, and the + evaluator treats series *presence* as "condition holds" (exactly PromQL's + alert semantics). + +## Example test cases + +`Cardano.Tracer.Test.Alarms.TimeseriesTests` instantiates this pattern with +three rules ported from the Grafana catalogue +(`grafana-alerts-as-timeseries-rules.md`), each driven once with data that +must not raise an alarm and once with data that must raise exactly one — six +cases total. Samples arrive every 30 s from `t0`, one evaluation round per +30 simulated seconds starting at round 1, and the history is read back +through `readHistoryFiltered` for inspection. + +| Rule (query construct) | Quiet case — why no alarm | Alarm case — publish round | +| --- | --- | --- | +| `mempool-high` — `cardano_node_metrics_txsInMempool_int now > 200`, for 600 s | mempool hovers at 120–180: expression never true | constant 250: true from round 1, `for` satisfied at round 21; extra rounds prove the edge publishes once | +| `blockheight-unchanged` — `rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0`, for 120 s | chain grows by 1 block per sample: rate > 0, series filtered out every round | constant block height: rate exactly 0, publishes at round 5 with severity `critical` | +| `high-ping-latency` — `avg_over_time (…ping_latency…[now - 5m; now]) > 500`, for 3600 s | 10 min spike at 600 ms, then recovery to 100 ms: expression true far shorter than `for`, pending state resets | constant 800 ms for 62 min: publishes at round 121 | + +The three quiet cases deliberately cover the three distinct ways a rule stays +silent: the expression is never true (mempool), the series is filtered out of +the result (blockheight), and the expression is true for less than the `for` +duration before recovering (ping latency). The alarm cases assert exactly one +event and inspect its `ruleId`, `severity`, `source` (`timeseries`), the +`node_name` label carried over from the series key, and the +per-series `sourceEventId` prefix (`ts::node_name=node-1:`). + +## Level 2: end-to-end with a launched `cardano-tracer` + +One smoke test should cover the full wiring: config parsing, store, evaluator +thread, alarm history endpoint. The pattern is the one `Test.Logs` uses: +start the tracer and a forwarder in-process inside a tasty test, then assert +over HTTP. + +Configuration for the test (YAML): + +```yaml +networkMagic: 42 +network: + acceptAt: "tracer.sock" +logging: + - logRoot: "logs" + logMode: FileMode + logFormat: ForMachine +hasTimeseries: + epHost: "127.0.0.1" + epPort: 3300 +alarms: + endpoint: + epHost: "127.0.0.1" + epPort: 3210 + allowInsecure: true # test only -- no TLS certificate + authentication: + producers: [] + readers: + - name: test-reader + tokenFile: "reader.token" # written by the test setup + allowHistory: true + timeseriesRules: + - ruleId: high-latency + summary: "latency above threshold" + severity: warning + query: "avg_over_time (ping_latency_ms[now - 5m; now]) > 500" + evaluateEvery: 1 # seconds; keep the test fast + consumers: [] +``` + +Feeding data -- two workable options: + +1. **Reuse the test forwarder** (`Cardano.Tracer.Test.Forwarder`, also + available as the `demo-forwarder` executable). It forwards EKG metrics over + `trace-forward` exactly like a node, and the acceptor inserts them into the + timeseries store with the `node_name` label. Extend it (or configure it) to + register a gauge with the metric name the rule queries. Caveats: metric + names pass through `sanitiseMetricName`, only numeric values are inserted, + and timestamps are "now" -- so the test asserts *eventually* (poll with a + deadline), not at exact instants. + +2. **A script cannot insert directly** -- there is no HTTP insert route, and + adding one for tests is not recommended (it would create a second, + test-only ingestion path that production never exercises). If a scriptable + entry point is ever needed, prefer a small Haskell driver that links the + forwarder library rather than a raw HTTP endpoint. + +Asserting: + +```bash +# data arrived? +curl 'http://127.0.0.1:3300/timeseries/query?query=ping_latency_ms' +# alarm raised? +curl -H "Authorization: Bearer $(cat reader.token)" \ + 'http://127.0.0.1:3210/alarms/v1/events?minSeverity=warning' +``` + +In a tasty test, do the same with a few lines of `http-client` and poll until +the history response contains the expected `ruleId` or a timeout expires. + +## What about preloading test data via the configuration? + +A config option like `hasTimeseries: { preloadFile: testdata.json }` -- a file +of `(labels, timestamp, metric, value)` rows loaded into the store right after +`Timeseries.create` in `Run.hs` -- would make end-to-end tests fully +deterministic (fixed timestamps, no forwarder needed) and is cheap to build. +It does not exist yet. If Level-2 tests become flaky because of the +"eventually" polling, this is the first improvement to make; until then, +Level 1 covers determinism and Level 2 covers wiring. + +## Summary + +| Level | What it tests | Data entry | Determinism | +| --- | --- | --- | --- | +| 1: in-process | query + rule semantics | `Timeseries.insert`, fixed timestamps | full | +| 2: launched tracer | config, threads, HTTP, history | test forwarder (EKG metrics) | poll with deadline | + +Put Level-1 properties in `Cardano.Tracer.Test.Alarms.Tests` (they need no +work directory) and the Level-2 smoke test in its own module following +`Test.Logs`' `propRunInLogsStructure` pattern. diff --git a/cardano-tracer/src/Cardano/Tracer/Configuration.hs b/cardano-tracer/src/Cardano/Tracer/Configuration.hs index 1803bc00968..5e22edd5aae 100644 --- a/cardano-tracer/src/Cardano/Tracer/Configuration.hs +++ b/cardano-tracer/src/Cardano/Tracer/Configuration.hs @@ -11,6 +11,14 @@ module Cardano.Tracer.Configuration ( Address + , AlarmFilterConfig (..) + , AlarmsAuthConfig (..) + , AlarmsConfig (..) + , AlarmsConsumerConfig (..) + , AlarmsLimitsConfig (..) + , AlarmsRetentionConfig (..) + , AlarmsTimeseriesRuleConfig (..) + , AlarmsTraceRuleConfig (..) , Certificate (..) , Net.HowToConnect (..) , Endpoint (..) @@ -20,18 +28,22 @@ module Cardano.Tracer.Configuration , LogMode (..) , LoggingParams (..) , Network (..) + , ProducerCredentialConfig (..) + , ReaderCredentialConfig (..) , RotationParams (..) , TracerConfig (..) , Verbosity (..) + , alarmSeverityToText + , parseAlarmSeverityText , readTracerConfig ) where -import Cardano.Logging.Types (HowToConnect) +import Cardano.Logging.Types (HowToConnect, SeverityS (..)) import qualified Cardano.Logging.Types as Log import qualified Cardano.Logging.Types as Net import Control.Applicative ((<|>)) -import Data.Aeson (FromJSON (..), ToJSON (..), withObject, (.:)) +import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.:?), (.=)) import Data.Fixed (Pico) import Data.Function ((&)) import Data.Functor ((<&>)) @@ -41,7 +53,7 @@ import Data.List.Extra (notNull) import Data.List.NonEmpty (NonEmpty) import qualified Data.List.NonEmpty as NE import Data.Map.Strict (Map) -import Data.Maybe (catMaybes) +import Data.Maybe (catMaybes, fromMaybe) import Data.String (fromString) import Data.Text (Text) import qualified Data.Text as Text @@ -106,6 +118,224 @@ instance FromJSON RotationParams where rpKeepFilesNum <- o .: "rpKeepFilesNum" pure RotationParams{..} +-- | A conjunctive selection over alarm events, used both for a static +-- consumer's configuration and for a reader credential's allowed +-- (ceiling) or requested filter. +data AlarmFilterConfig = AlarmFilterConfig + { afcSource :: !(Maybe Text) + , afcRuleId :: !(Maybe Text) + , afcMinSeverity :: !(Maybe SeverityS) + , afcScope :: !(Maybe (Map Text Text)) + , afcLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmFilterConfig where + parseJSON = withObject "AlarmFilterConfig" \o -> do + afcSource <- o .:? "source" + afcRuleId <- o .:? "ruleId" + afcMinSeverityText <- o .:? "minSeverity" + afcMinSeverity <- traverse parseSeverityOrFail afcMinSeverityText + afcScope <- o .:? "scope" + afcLabels <- o .:? "labels" + pure AlarmFilterConfig{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +instance ToJSON AlarmFilterConfig where + toJSON AlarmFilterConfig{..} = object $ catMaybes + [ ("source" .=) <$> afcSource + , ("ruleId" .=) <$> afcRuleId + , ("minSeverity" .=) . alarmSeverityToText <$> afcMinSeverity + , ("scope" .=) <$> afcScope + , ("labels" .=) <$> afcLabels + ] + +-- | A statically configured alarm dispatch target. The only variant +-- implemented so far is @log@; @webhook@/@email@ are the obvious follow-up +-- extension (see @Cardano.Tracer.Handlers.Alarms.Consumers@). +data AlarmsConsumerConfig = AlarmsConsumerLog + { aclName :: !Text + , aclEnabled :: !Bool + , aclFilter :: !(Maybe AlarmFilterConfig) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsConsumerConfig where + parseJSON = withObject "AlarmsConsumerConfig" \o -> do + consumerType :: Text <- o .: "type" + case consumerType of + "log" -> AlarmsConsumerLog + <$> o .: "name" + <*> (fromMaybe True <$> o .:? "enabled") + <*> o .:? "filter" + other -> fail ("unknown alarm consumer type: " <> Text.unpack other) + +instance ToJSON AlarmsConsumerConfig where + toJSON AlarmsConsumerLog{..} = object + [ "type" .= ("log" :: Text) + , "name" .= aclName + , "enabled" .= aclEnabled + , "filter" .= aclFilter + ] + +-- | A producer credential: a bearer token (read once from @pcTokenFile@) that +-- authenticates alarm ingress requests as coming from @pcSource@. The +-- source is never taken from the request body itself. +data ProducerCredentialConfig = ProducerCredentialConfig + { pcName :: !Text + , pcTokenFile :: !FilePath + , pcSource :: !Text + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | A reader credential: a bearer token that grants history access, capped +-- by an allowed (ceiling) filter that a caller's requested filter may +-- only narrow. +data ReaderCredentialConfig = ReaderCredentialConfig + { rcName :: !Text + , rcTokenFile :: !FilePath + , rcAllowHistory :: !(Maybe Bool) + , rcFilter :: !(Maybe AlarmFilterConfig) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +data AlarmsAuthConfig = AlarmsAuthConfig + { aacProducers :: ![ProducerCredentialConfig] + , aacReaders :: ![ReaderCredentialConfig] + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +data AlarmsRetentionConfig = AlarmsRetentionConfig + { arcMaxAgeSeconds :: !(Maybe Word64) + , arcMaxEvents :: !(Maybe Word64) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +newtype AlarmsLimitsConfig = AlarmsLimitsConfig + { alcMaxEventBytes :: Maybe Word64 + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | A rule that raises an alarm for every received trace message whose +-- severity is at or above @threshold@. @suppressForSecs@ (default 300) +-- bounds the alarm frequency: within one window, repeated matches from the +-- same node and namespace collapse into a single alarm via the store's +-- idempotency key (see @Cardano.Tracer.Handlers.Alarms.TraceRules@). +data AlarmsTraceRuleConfig = AlarmsTraceRuleConfig + { atrRuleId :: !Text + , atrSummary :: !(Maybe Text) + , atrThreshold :: !SeverityS + , atrSuppressForSecs :: !(Maybe Word64) + , atrLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsTraceRuleConfig where + parseJSON = withObject "AlarmsTraceRuleConfig" \o -> do + atrRuleId <- o .: "ruleId" + atrSummary <- o .:? "summary" + thresholdText <- o .: "threshold" + atrThreshold <- parseSeverityOrFail thresholdText + atrSuppressForSecs <- o .:? "suppressForSecs" + atrLabels <- o .:? "labels" + pure AlarmsTraceRuleConfig{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +instance ToJSON AlarmsTraceRuleConfig where + toJSON AlarmsTraceRuleConfig{..} = object $ + [ "ruleId" .= atrRuleId + , "threshold" .= alarmSeverityToText atrThreshold + ] <> catMaybes + [ ("summary" .=) <$> atrSummary + , ("suppressForSecs" .=) <$> atrSuppressForSecs + , ("labels" .=) <$> atrLabels + ] + +-- | A rule that periodically evaluates a @cardano-timeseries-io@ query and +-- raises an alarm when the query returns 'Truth' (or a truthy +-- 'InstantVector' entry) for at least @for@ seconds. The @sourceEventId@ +-- embeds the rule id and a canonical series key from the sample's labels +-- so per-series edges are deduplicated by the store. +data AlarmsTimeseriesRuleConfig = AlarmsTimeseriesRuleConfig + { atsRuleId :: !Text + , atsSummary :: !(Maybe Text) + , atsSeverity :: !SeverityS + , atsQuery :: !Text + , atsEvaluateEvery :: !Word64 -- ^ seconds between evaluations + , atsFor :: !(Maybe Word64) -- ^ seconds the sample must stay truthy before publishing + , atsRepeatEvery :: !(Maybe Word64) -- ^ seconds between reminder alarms while still true + , atsLabels :: !(Maybe (Map Text Text)) + } + deriving stock (Eq, Show, Generic) + +instance FromJSON AlarmsTimeseriesRuleConfig where + parseJSON = withObject "AlarmsTimeseriesRuleConfig" \o -> do + atsRuleId <- o .: "ruleId" + atsSummary <- o .:? "summary" + severityText <- o .: "severity" + atsSeverity <- maybe (fail ("unknown severity: " <> Text.unpack severityText)) pure + (parseAlarmSeverityText severityText) + atsQuery <- o .: "query" + atsEvaluateEvery <- o .: "evaluateEvery" + atsFor <- o .:? "for" + atsRepeatEvery <- o .:? "repeatEvery" + atsLabels <- o .:? "labels" + pure AlarmsTimeseriesRuleConfig{..} + +instance ToJSON AlarmsTimeseriesRuleConfig where + toJSON AlarmsTimeseriesRuleConfig{..} = object $ + [ "ruleId" .= atsRuleId + , "severity" .= alarmSeverityToText atsSeverity + , "query" .= atsQuery + , "evaluateEvery" .= atsEvaluateEvery + ] <> catMaybes + [ ("summary" .=) <$> atsSummary + , ("for" .=) <$> atsFor + , ("repeatEvery" .=) <$> atsRepeatEvery + , ("labels" .=) <$> atsLabels + ] + +-- | Configuration for the alarm subsystem (see +-- @cardano-tracer/docs/alarm-system-concept.md@). @Nothing@ for the +-- enclosing 'Maybe' in 'TracerConfig' is the on/off switch for the whole +-- subsystem; every field below is only meaningful once it's turned on. +data AlarmsConfig = AlarmsConfig + { alEndpoint :: !Endpoint + , alAllowInsecure :: !(Maybe Bool) + , alRetention :: !(Maybe AlarmsRetentionConfig) + , alLimits :: !(Maybe AlarmsLimitsConfig) + , alAuthentication :: !AlarmsAuthConfig + , alConsumers :: ![AlarmsConsumerConfig] + , alTraceRules :: !(Maybe [AlarmsTraceRuleConfig]) + , alTimeseriesRules :: !(Maybe [AlarmsTimeseriesRuleConfig]) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (FromJSON, ToJSON) + +-- | The alarm envelope's wire form uses lowercase severity names (e.g. +-- @"critical"@), unlike 'SeverityS'\'s own derived 'FromJSON'\/'ToJSON' +-- instances (from @trace-dispatcher@), which serialise the capitalised +-- constructor name (@"Critical"@). Every place a severity crosses the +-- alarm system's wire boundary -- the 'AlarmEvent' envelope, YAML +-- 'AlarmFilterConfig', and the @minSeverity=@ query parameter -- must go +-- through these two functions instead of 'SeverityS'\'s own instances, or +-- producers/consumers will silently disagree on casing. +alarmSeverityToText :: SeverityS -> Text +alarmSeverityToText = Text.toLower . Text.pack . show + +parseAlarmSeverityText :: Text -> Maybe SeverityS +parseAlarmSeverityText t = + lookup (Text.toLower t) [(alarmSeverityToText s, s) | s <- [minBound .. maxBound]] + -- | Logging mode. data LogMode = FileMode -- ^ Store items in log file. @@ -164,6 +394,7 @@ data TracerConfig = TracerConfig , hasEKG :: !(Maybe Endpoint) -- ^ Endpoint for EKG web-page. , hasPrometheus :: !(Maybe Endpoint) -- ^ Endpoint for Prometheus web-page. , hasTimeseries :: !(Maybe Endpoint) + , alarms :: !(Maybe AlarmsConfig) -- ^ Alarm subsystem configuration; 'Nothing' disables it. , tlsCertificate :: !(Maybe Certificate) -- | Socket for tracer's to reforward on. Second member of the triplet is the list of prefixes to reforward. -- Third member of the triplet is the forwarder config. @@ -205,6 +436,8 @@ wellFormed TracerConfig { network , hasEKG , hasPrometheus + , hasTimeseries + , alarms , logging } = if null problems @@ -220,15 +453,34 @@ wellFormed TracerConfig , check "duplicate ports in config" $ hasDuplicates ports , check "no host(s) in hasEKG" . nullEndpoint =<< hasEKG , check "no host in hasPrometheus" . nullEndpoint =<< hasPrometheus + , check "no host in hasTimeseries" . nullEndpoint =<< hasTimeseries + , check "no host in alarms endpoint" . nullEndpoint . alEndpoint =<< alarms + , check "alarms: no producer or reader credentials configured" . noCredentials =<< alarms + , check "alarms: duplicate consumer names" . hasDuplicateConsumerNames =<< alarms ] + -- NB. every internal service's endpoint port is included here, including + -- 'hasTimeseries' and 'alarms' (the former was previously missing from + -- this check). ports :: [Port] - ports = epPort <$> catMaybes [hasEKG, hasPrometheus] + ports = epPort <$> catMaybes + [hasEKG, hasPrometheus, hasTimeseries, alEndpoint <$> alarms] check :: String -> Bool -> Maybe String check msg True = Just msg check _ False = Nothing + noCredentials :: AlarmsConfig -> Bool + noCredentials AlarmsConfig{alAuthentication = AlarmsAuthConfig{aacProducers, aacReaders}} = + null aacProducers && null aacReaders + + hasDuplicateConsumerNames :: AlarmsConfig -> Bool + hasDuplicateConsumerNames AlarmsConfig{alConsumers} = + hasDuplicates (map consumerName alConsumers) + + consumerName :: AlarmsConsumerConfig -> Text + consumerName AlarmsConsumerLog{aclName} = aclName + nullAddress :: Address -> Bool nullAddress (Net.LocalPipe address) = null address nullAddress (Net.RemoteSocket host _port) = Text.null host diff --git a/cardano-tracer/src/Cardano/Tracer/Environment.hs b/cardano-tracer/src/Cardano/Tracer/Environment.hs index 5ea9ef3fecb..e6708a72a66 100644 --- a/cardano-tracer/src/Cardano/Tracer/Environment.hs +++ b/cardano-tracer/src/Cardano/Tracer/Environment.hs @@ -5,6 +5,7 @@ module Cardano.Tracer.Environment import Cardano.Logging.Types import Cardano.Timeseries.Component (TimeseriesHandle) import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry (AlarmRegistry) import Cardano.Tracer.MetaTrace import Cardano.Tracer.Types @@ -29,4 +30,5 @@ data TracerEnv = TracerEnv , teStateDir :: !(Maybe FilePath) , teMetricsHelp :: ![(Text, Builder)] , teTimeseriesHandle :: !(Maybe TimeseriesHandle) + , teAlarmRegistry :: !(Maybe AlarmRegistry) } diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs new file mode 100644 index 00000000000..da4f38fdd56 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Auth.hs @@ -0,0 +1,96 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Minimal bearer-token authentication for the alarm HTTP surface. There is +-- no existing auth pattern anywhere in @cardano-tracer@ to follow (none of +-- the existing Prometheus\/EKG\/Timeseries servers check any credential at +-- all), so this is deliberately simple: static token tables, read once from +-- config-supplied files. No hashing, no constant-time comparison, no +-- rotation, no rate limiting. +module Cardano.Tracer.Handlers.Alarms.Auth + ( ProducerCredential (..) + , ReaderCredential (..) + , AuthTables (..) + , loadCredentials + , bearerToken + , lookupProducer + , lookupReader + ) where + +import Cardano.Tracer.Configuration (AlarmsAuthConfig (..), ProducerCredentialConfig (..), + ReaderCredentialConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types + +import qualified Data.ByteString.Char8 as BSC +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Encoding.Error as TE +import Network.HTTP.Types (hAuthorization) +import Network.Wai (Request, requestHeaders) + +data ProducerCredential = ProducerCredential + { pcrSource :: !AlarmSource + } + deriving stock (Eq, Show) + +data ReaderCredential = ReaderCredential + { rcrName :: !Text + , rcrAllowHistory :: !Bool + , rcrFilter :: !AlarmFilter + } + deriving stock (Eq, Show) + +data AuthTables = AuthTables + { atProducers :: !(Map Text ProducerCredential) + , atReaders :: !(Map Text ReaderCredential) + } + +-- | Reads every configured token file once, at registry-construction time +-- (per the design doc's Security section: "Secrets should be read from +-- protected files"). Not part of 'wellFormed', since that's pure and this +-- needs 'IO'. +loadCredentials :: AlarmsAuthConfig -> IO AuthTables +loadCredentials AlarmsAuthConfig{aacProducers, aacReaders} = do + producers <- traverse loadProducer aacProducers + readers <- traverse loadReader aacReaders + pure AuthTables + { atProducers = Map.fromList producers + , atReaders = Map.fromList readers + } + where + loadProducer ProducerCredentialConfig{pcTokenFile, pcSource} = do + token <- readTokenFile pcTokenFile + pure (token, ProducerCredential (AlarmSource pcSource)) + + loadReader ReaderCredentialConfig{rcName, rcTokenFile, rcAllowHistory, rcFilter} = do + token <- readTokenFile rcTokenFile + pure ( token + , ReaderCredential + { rcrName = rcName + , rcrAllowHistory = fromMaybe False rcAllowHistory + , rcrFilter = maybe emptyAlarmFilter filterFromConfig rcFilter + } + ) + +readTokenFile :: FilePath -> IO Text +readTokenFile path = Text.strip . Text.pack <$> readFile path + +-- | Extract the bearer token from a request's @Authorization@ header, if +-- present and well-formed. +bearerToken :: Request -> Maybe Text +bearerToken req = lookup hAuthorization (requestHeaders req) >>= extractBearer + where + extractBearer bs + | BSC.isPrefixOf "Bearer " bs = + Just (Text.strip (TE.decodeUtf8With TE.lenientDecode (BSC.drop 7 bs))) + | otherwise = Nothing + +lookupProducer :: AuthTables -> Text -> Maybe ProducerCredential +lookupProducer tables token = Map.lookup token (atProducers tables) + +lookupReader :: AuthTables -> Text -> Maybe ReaderCredential +lookupReader tables token = Map.lookup token (atReaders tables) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs new file mode 100644 index 00000000000..8405afc5c24 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Consumers.hs @@ -0,0 +1,46 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Statically configured alarm dispatch targets. Phase 1 (this sketch) +-- implements only 'ConsumerLog', matching the design doc's own staging +-- ("Add a log consumer to exercise filtering and dispatch"). Adding +-- @webhook@\/@email@ consumers later is one new constructor here and one new +-- case in 'dispatch' -- @email@ in particular can reuse the SMTP-sending +-- primitives in "Cardano.Tracer.Handlers.Notifications.Email" -- not +-- speculative machinery added now. +module Cardano.Tracer.Handlers.Alarms.Consumers + ( AlarmConsumer (..) + , consumerFromConfig + , dispatch + ) where + +import Cardano.Tracer.Configuration (AlarmsConsumerConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) + +import Data.Text (Text) + +data AlarmConsumer = ConsumerLog + { clName :: !Text + , clEnabled :: !Bool + , clFilter :: !AlarmFilter + } + +consumerFromConfig :: AlarmsConsumerConfig -> AlarmConsumer +consumerFromConfig AlarmsConsumerLog{aclName, aclEnabled, aclFilter} = ConsumerLog + { clName = aclName + , clEnabled = aclEnabled + , clFilter = maybe emptyAlarmFilter filterFromConfig aclFilter + } + +dispatch :: Trace IO TracerTrace -> AlarmConsumer -> AlarmEvent -> IO () +dispatch tracer consumer@ConsumerLog{clEnabled, clFilter} ev + | clEnabled && matchesFilter clFilter ev = + traceWith tracer TracerAlarmDispatched + { ttAlarmDispatchedConsumer = clName consumer + , ttAlarmDispatchedSource = unAlarmSource (source ev) + , ttAlarmDispatchedRuleId = unRuleId (ruleId ev) + , ttAlarmDispatchedSeverity = severity ev + , ttAlarmDispatchedSummary = summary ev + } + | otherwise = pure () diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs new file mode 100644 index 00000000000..d5bd75e45be --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Registry.hs @@ -0,0 +1,179 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | The central alarm registry: static consumers (fixed at startup) sitting +-- on top of the append-only store. Mirrors the brainstorm design doc's +-- @AlarmRegistry@ sketch. Like +-- "Cardano.Tracer.Handlers.Alarms.Types"\/"...Store", this module never +-- imports "Cardano.Tracer.Environment". +module Cardano.Tracer.Handlers.Alarms.Registry + ( AlarmRegistry + , newAlarmRegistry + , acceptEvent + , rejectEvent + , checkTraceObjectsForAlarms + , readHistoryFiltered + , runTimeseriesEvaluator + , evaluateTimeseriesRule + , evaluateTimeseriesRules + , traceHistoryRead + , lookupProducerCredential + , lookupReaderCredential + ) where + +import Cardano.Logging.Types (TraceObject) +import Cardano.Timeseries.AsText (asText) +import Cardano.Timeseries.Component (TimeseriesHandle, execute) +import Cardano.Tracer.Configuration (AlarmsConfig (..), AlarmsRetentionConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Auth +import Cardano.Tracer.Handlers.Alarms.Consumers +import Cardano.Tracer.Handlers.Alarms.Store +import Cardano.Tracer.Handlers.Alarms.TimeseriesRules +import Cardano.Tracer.Handlers.Alarms.TraceRules (TraceAlarmRule, traceAlarmSource, + traceRuleFromConfig, traceRuleRequest) +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TracerTrace (..), Trace, traceWith) + +import Control.Concurrent (threadDelay) +import Control.Concurrent.Async (async, link) +import Control.Monad (forever) +import Data.Foldable (for_) +import Data.Maybe (fromMaybe, mapMaybe) +import Data.Text (Text) +import Data.Time.Clock (UTCTime, getCurrentTime) +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) + +data AlarmRegistry = AlarmRegistry + { arStore :: !AlarmStoreHandle + , arConsumers :: ![AlarmConsumer] -- ^ static, fixed at startup + , arTraceRules :: ![TraceAlarmRule] + , arTimeseriesRules :: ![TimeseriesAlarmRule] + , arAuth :: !AuthTables + , arTracer :: !(Trace IO TracerTrace) + } + +newAlarmRegistry :: Trace IO TracerTrace -> AlarmsConfig -> IO AlarmRegistry +newAlarmRegistry tracer AlarmsConfig{alRetention, alConsumers, alAuthentication, alTraceRules, alTimeseriesRules} = do + store <- newAlarmStore (fromMaybe emptyRetention alRetention) + authTable <- loadCredentials alAuthentication + tsRules <- traverse timeseriesRuleFromConfig (fromMaybe [] alTimeseriesRules) + pure AlarmRegistry + { arStore = store + , arConsumers = map consumerFromConfig alConsumers + , arTraceRules = map traceRuleFromConfig (fromMaybe [] alTraceRules) + , arTimeseriesRules = tsRules + , arAuth = authTable + , arTracer = tracer + } + where + emptyRetention = AlarmsRetentionConfig Nothing Nothing + +-- | Accept a (possibly duplicate) producer submission. On a genuinely new +-- event: insert into the store, then synchronously dispatch to every +-- static consumer. On a replay of an already-accepted +-- @(source, sourceEventId)@: skip dispatch -- "does not dispatch it +-- twice", per the design doc. +acceptEvent :: AlarmRegistry -> AlarmSource -> IngressRequest -> IO (AlarmEvent, Bool) +acceptEvent registry src req = do + receivedAt <- getCurrentTime + (_cursor, ev, wasCreated) <- insertOrGetExisting (arStore registry) src receivedAt req + if wasCreated + then do + traceWith (arTracer registry) TracerAlarmAccepted + { ttAlarmAcceptedSource = unAlarmSource src + , ttAlarmAcceptedRuleId = unRuleId (ruleId ev) + , ttAlarmAcceptedSeverity = severity ev + } + for_ (arConsumers registry) \consumer -> dispatch (arTracer registry) consumer ev + else + traceWith (arTracer registry) TracerAlarmDuplicate + { ttAlarmDuplicateSource = unAlarmSource src + , ttAlarmDuplicateSourceEventId = sourceEventId ev + } + pure (ev, wasCreated) + +-- | Trace a rejected (invalid, unauthenticated, or oversized) ingress +-- request. Never goes through 'acceptEvent' -- a rejected request never +-- reaches the store. +rejectEvent :: AlarmRegistry -> Text -> IO () +rejectEvent registry reason = + traceWith (arTracer registry) TracerAlarmRejected { ttAlarmRejectedReason = reason } + +-- | Check every received trace message against the configured trace +-- severity rules and submit each match through the normal 'acceptEvent' +-- path. Within a rule's suppression window, repeated matches share their +-- idempotency key, so 'acceptEvent' reports them as duplicates and +-- dispatches nothing. +checkTraceObjectsForAlarms :: AlarmRegistry -> Text -> [TraceObject] -> IO () +checkTraceObjectsForAlarms registry nodeName traceObjects = + for_ (arTraceRules registry) \rule -> + for_ (mapMaybe (traceRuleRequest rule nodeName) traceObjects) \req -> + acceptEvent registry traceAlarmSource req + +readHistoryFiltered :: AlarmRegistry -> Maybe AlarmCursor -> Int -> AlarmFilter -> IO [(AlarmCursor, AlarmEvent)] +readHistoryFiltered registry = readHistory (arStore registry) + +-- | Emit a meta-trace recording a successful history read: which reader +-- credential fired it and how many events the response carried. Consumers +-- of the ingress path have 'TracerAlarmAccepted'/'TracerAlarmDuplicate'; +-- this is the corresponding observability hook for the read path. +traceHistoryRead :: AlarmRegistry -> Text -> Int -> IO () +traceHistoryRead registry readerName resultCount = + traceWith (arTracer registry) TracerAlarmHistoryRead + { ttAlarmHistoryReader = readerName + , ttAlarmHistoryResultCount = resultCount + } + +-- | Spawn one supervised thread per timeseries rule. Each ticks on the +-- rule's evaluateEvery interval and pushes any resulting alarms +-- through 'acceptEvent'. +runTimeseriesEvaluator :: AlarmRegistry -> TimeseriesHandle -> IO () +runTimeseriesEvaluator registry tsHandle = + for_ (arTimeseriesRules registry) \rule -> do + a <- async (evaluatorLoop registry tsHandle rule) + link a + +evaluatorLoop :: AlarmRegistry -> TimeseriesHandle -> TimeseriesAlarmRule -> IO () +evaluatorLoop registry tsHandle rule = forever $ do + threadDelay (fromIntegral (tarEvaluateEvery rule) * 1_000_000) + now <- getCurrentTime + evaluateTimeseriesRule registry tsHandle now rule + +-- | Evaluate one rule at an explicit timestamp: run its query against +-- the store, decode the samples, advance the per-series state +-- machines, and push any resulting alarms through 'acceptEvent'. +-- Query and shape failures are traced as evaluation failures, never +-- published as alarms. The evaluation time is an argument (rather +-- than 'getCurrentTime') so tests can drive rules deterministically; +-- the production 'evaluatorLoop' passes the current time. +evaluateTimeseriesRule :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> TimeseriesAlarmRule -> IO () +evaluateTimeseriesRule registry tsHandle now rule = do + result <- execute tsHandle (round (utcTimeToPOSIXSeconds now * 1000)) (tarQuery rule) + case result of + Left err -> + failed (asText err) + Right val -> case decodeSamples val of + Nothing -> failed "unexpected value shape from query" + Just samples -> do + reqs <- evaluateOnce rule now samples + for_ reqs \req -> acceptEvent registry timeseriesAlarmSource req + where + failed :: Text -> IO () + failed reason = + traceWith (arTracer registry) TracerAlarmTimeseriesEvalFailed + { ttAlarmTimeseriesEvalFailedRule = unRuleId (tarRuleId rule) + , ttAlarmTimeseriesEvalFailedReason = reason + } + +-- | Evaluate every configured timeseries rule once at the given +-- timestamp. +evaluateTimeseriesRules :: AlarmRegistry -> TimeseriesHandle -> UTCTime -> IO () +evaluateTimeseriesRules registry tsHandle now = + for_ (arTimeseriesRules registry) (evaluateTimeseriesRule registry tsHandle now) + +lookupProducerCredential :: AlarmRegistry -> Text -> Maybe ProducerCredential +lookupProducerCredential registry = lookupProducer (arAuth registry) + +lookupReaderCredential :: AlarmRegistry -> Text -> Maybe ReaderCredential +lookupReaderCredential registry = lookupReader (arAuth registry) diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs new file mode 100644 index 00000000000..13593e4dfaf --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Server.hs @@ -0,0 +1,222 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | The alarm HTTP surface: producer ingress and history. A hand-rolled +-- 'wai' 'Application', modeled directly on +-- "Cardano.Tracer.Handlers.Metrics.TimeseriesServer" (no @servant@ +-- anywhere in this codebase). +module Cardano.Tracer.Handlers.Alarms.Server + ( runAlarms + ) where + +import Cardano.Tracer.Configuration +import Cardano.Tracer.Environment (TracerEnv (..)) +import Cardano.Tracer.Handlers.Alarms.Auth +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.Handlers.Metrics.Utils (contentHdrJSON) +import Cardano.Tracer.MetaTrace + +import Control.Monad (join) +import Data.Aeson (encode, object, (.=)) +import qualified Data.Aeson as Aeson +import qualified Data.ByteString as BS +import qualified Data.ByteString.Builder as BB +import qualified Data.ByteString.Lazy as BL +import Data.Foldable (for_) +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Text.Read (decimal) +import Data.Word (Word64) +import Network.HTTP.Types +import Network.Wai +import Network.Wai.Handler.Warp hiding (run) +import Network.Wai.Handler.WarpTLS +import System.Time.Extra (sleep) + +-------------------------------------------------------------------------------- +-- Response helpers +-------------------------------------------------------------------------------- + +jsonResponse :: Status -> Aeson.Value -> Response +jsonResponse st v = responseLBS st contentHdrJSON (encode v) + +okResponse, createdResponse :: Aeson.Value -> Response +okResponse = jsonResponse status200 +createdResponse = jsonResponse status201 + +badRequest :: Text -> Response +badRequest msg = jsonResponse status400 (object ["error" .= msg]) + +unauthorized :: Response +unauthorized = jsonResponse status401 (object ["error" .= ("missing or invalid bearer token" :: Text)]) + +forbidden :: Text -> Response +forbidden msg = jsonResponse status403 (object ["error" .= msg]) + +notFound :: Response +notFound = responseLBS status404 [] "" + +-------------------------------------------------------------------------------- +-- Config helpers +-------------------------------------------------------------------------------- + +maxEventBytes :: AlarmsConfig -> Word64 +maxEventBytes cfg = fromMaybe 262144 (alLimits cfg >>= alcMaxEventBytes) -- 256 KiB default, matching the design doc's example config + +-------------------------------------------------------------------------------- +-- Query-string / header helpers +-------------------------------------------------------------------------------- + +lookupParam :: Text -> [(Text, Maybe Text)] -> Maybe Text +lookupParam key params = join (lookup key params) + +parseCursorText :: Text -> Maybe AlarmCursor +parseCursorText t = case decimal t of + Right (n, "") -> Just (AlarmCursor n) + _ -> Nothing + +parseIntText :: Text -> Maybe Int +parseIntText t = case decimal t of + Right (n, "") -> Just n + _ -> Nothing + +cursorToText :: AlarmCursor -> Text +cursorToText (AlarmCursor n) = Text.pack (show n) + +requestedFilterFromQuery :: [(Text, Maybe Text)] -> AlarmFilter +requestedFilterFromQuery params = AlarmFilter + { afSource = AlarmSource <$> lookupParam "source" params + , afRuleId = RuleId <$> lookupParam "ruleId" params + , afMinSeverity = lookupParam "minSeverity" params >>= parseAlarmSeverityText + -- Scope/label query-string filtering isn't implemented in this sketch; + -- only source/ruleId/minSeverity are selectable via GET requests. + , afScope = mempty + , afLabels = mempty + } + +-------------------------------------------------------------------------------- +-- Body-size-limited read (checked before decoding, per the design doc's +-- ingress limits -- never fully materialise an attacker-controlled oversized +-- payload) +-------------------------------------------------------------------------------- + +readLimitedBody :: Word64 -> Request -> IO (Maybe BL.ByteString) +readLimitedBody limit req = case requestBodyLength req of + KnownLength n | n > limit -> pure Nothing + _ -> go 0 mempty + where + go total acc = do + chunk <- getRequestBodyChunk req + let total' = total + fromIntegral (BS.length chunk) + if BS.null chunk + then pure (Just (BB.toLazyByteString acc)) + else if total' > limit + then pure Nothing + else go total' (acc <> BB.byteString chunk) + +-------------------------------------------------------------------------------- +-- Routes +-------------------------------------------------------------------------------- + +alarmsApp :: AlarmsConfig -> AlarmRegistry -> Application +alarmsApp cfg registry request send = case (requestMethod request, pathInfo request) of + ("POST", ["alarms", "v1", "events"]) -> handleIngress cfg registry request send + ("GET", ["alarms", "v1", "events"]) -> handleHistory registry request send + _ -> send notFound + +handleIngress :: AlarmsConfig -> AlarmRegistry -> Application +handleIngress cfg registry request send = + case bearerToken request >>= lookupProducerCredential registry of + Nothing -> send unauthorized + Just (ProducerCredential src) -> do + readLimitedBody (maxEventBytes cfg) request >>= \case + Nothing -> do + rejectEvent registry "request body missing or exceeds the configured size limit" + send (badRequest "request body missing or exceeds the configured size limit") + Just bs -> case Aeson.eitherDecode bs of + Left err -> do + rejectEvent registry (Text.pack ("invalid alarm event: " <> err)) + send (badRequest (Text.pack ("invalid alarm event: " <> err))) + Right ingressReq -> do + (ev, wasCreated) <- acceptEvent registry src ingressReq + send $ (if wasCreated then createdResponse else okResponse) $ object + [ "eventId" .= eventId ev + , "receivedAt" .= receivedAt ev + , "created" .= wasCreated + ] + +handleHistory :: AlarmRegistry -> Application +handleHistory registry request send = + case bearerToken request >>= lookupReaderCredential registry of + Nothing -> send unauthorized + Just credential + | not (rcrAllowHistory credential) -> send (forbidden "history access is not permitted for this credential") + | otherwise -> + let params = queryToQueryText (queryString request) + requested = requestedFilterFromQuery params + after = lookupParam "after" params >>= parseCursorText + limit = fromMaybe 100 (lookupParam "limit" params >>= parseIntText) + in if not (filterNarrows (rcrFilter credential) requested) + then send (forbidden "requested filter is not permitted by this credential") + else do + events <- readHistoryFiltered registry after limit requested + traceHistoryRead registry (rcrName credential) (length events) + send $ okResponse $ object + [ "events" .= map (Aeson.toJSON . snd) events + , "nextCursor" .= case events of + [] -> Nothing + _ -> Just (cursorToText (fst (last events))) + ] + +-------------------------------------------------------------------------------- +-- Server startup +-------------------------------------------------------------------------------- + +runAlarms :: TracerEnv -> IO () +runAlarms tracerEnv = + for_ ((,) <$> alarms (teConfig tracerEnv) <*> teAlarmRegistry tracerEnv) \(cfg, registry) -> + runAlarmsServer tracerEnv cfg registry + +-- | Unlike the existing Prometheus\/EKG\/Timeseries servers -- which fall +-- back to plaintext with just a warning trace if TLS was requested but no +-- certificate is configured -- the alarm server refuses to start in that +-- situation unless 'alAllowInsecure' is explicitly set. Alarm requests +-- carry bearer tokens in the clear otherwise, and the design doc's +-- Security section calls for refusing an externally reachable clear-text +-- endpoint by default. +runAlarmsServer :: TracerEnv -> AlarmsConfig -> AlarmRegistry -> IO () +runAlarmsServer tracerEnv cfg registry = do + -- Pause to prevent collision between "Listening"-notifications from servers. + sleep 0.3 + case (wantsSSL, tlsCertificate (teConfig tracerEnv)) of + (True, Nothing) | not insecureOk -> + traceWith (teTracer tracerEnv) TracerAlarmRejected + { ttAlarmRejectedReason = + "alarms endpoint requested TLS but no certificate is configured; " <> + "refusing to start (set alAllowInsecure: true to override)" + } + (True, Nothing) -> do + traceWith (teTracer tracerEnv) TracerMissingCertificate { ttMissingCertificateEndpoint = endpoint } + startPlain + (True, Just cert) -> do + traceWith (teTracer tracerEnv) TracerStartedAlarms { ttAlarmsEndpoint = endpoint } + runTLS (tlsSettingsFor cert) settings application + (False, _) -> do + traceWith (teTracer tracerEnv) TracerStartedAlarms { ttAlarmsEndpoint = endpoint } + startPlain + where + endpoint = alEndpoint cfg + insecureOk = fromMaybe False (alAllowInsecure cfg) + wantsSSL = epForceSSL endpoint == Just True + + settings = setEndpoint endpoint defaultSettings + application = alarmsApp cfg registry + + startPlain = runSettings settings application + + tlsSettingsFor Certificate{..} = + tlsSettingsChain certificateFile (fromMaybe [] certificateChain) certificateKeyFile diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs new file mode 100644 index 00000000000..ff6c4e2f260 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Store.hs @@ -0,0 +1,162 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | An in-memory, append-only alarm store. Deliberately not persisted to +-- disk: the design doc (@cardano-tracer/docs/alarm-system-concept.md@) +-- explicitly leaves the storage backend as an open decision, so history is +-- lost across a @cardano-tracer@ restart in this sketch. The concrete +-- backend is meant to sit behind this small handle so a persistent +-- implementation can be swapped in later without changing callers. +module Cardano.Tracer.Handlers.Alarms.Store + ( AlarmStoreHandle + , newAlarmStore + , insertOrGetExisting + , readHistory + , pruneOnce + ) where + +import Cardano.Tracer.Configuration (AlarmsRetentionConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.Time (getTimeMs) + +import Control.Concurrent (threadDelay) +import Control.Concurrent.Async (async, link, race_) +import Control.Concurrent.MVar (MVar, newEmptyMVar, takeMVar) +import Control.Concurrent.STM (atomically) +import Control.Concurrent.STM.TVar (TVar, modifyTVar', newTVarIO, readTVar, readTVarIO, + stateTVar, writeTVar) +import Control.Monad (forever) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, diffUTCTime, getCurrentTime) +import Data.Word (Word64) + +data AlarmStoreHandle = AlarmStoreHandle + { ashEvents :: !(TVar (Map AlarmCursor AlarmEvent)) + , ashIndex :: !(TVar (Map (AlarmSource, Text) AlarmCursor)) -- ^ (source, sourceEventId) -> cursor, for idempotency + , ashNextSeq :: !(TVar Word64) + , ashStartTag :: !Text -- ^ process-start timestamp (ms); prefixes generated eventIds + , ashRetention :: !AlarmsRetentionConfig + , ashWake :: !(MVar ()) -- ^ unused for now (no dynamic reconfiguration in this sketch); kept so the pruner loop matches Cardano.Timeseries.Component.create's shape + } + +-- | Prune every 60 seconds when any retention limit is configured. With no +-- retention there is nothing to prune, ever, and no pruner is started: a +-- thread parked forever on 'ashWake' would be killed by the RTS's deadlock +-- detector once the handle becomes garbage, and 'link' would then rethrow +-- into whatever the creating thread is doing by that time (observed as +-- spurious failures in unrelated tests). +pruneIntervalMicros :: Int +pruneIntervalMicros = 60 * 1000 * 1000 + +newAlarmStore :: AlarmsRetentionConfig -> IO AlarmStoreHandle +newAlarmStore retention = do + events <- newTVarIO Map.empty + index <- newTVarIO Map.empty + nextSeq <- newTVarIO 0 + wake <- newEmptyMVar + startTag <- Text.pack . show <$> getTimeMs + let handle = AlarmStoreHandle events index nextSeq startTag retention wake + case (arcMaxAgeSeconds retention, arcMaxEvents retention) of + (Nothing, Nothing) -> pure () + _ -> async (runPruner handle) >>= link + pure handle + where + runPruner :: AlarmStoreHandle -> IO () + runPruner handle = forever $ do + now <- getCurrentTime + pruneOnce handle now + race_ (threadDelay pruneIntervalMicros) (takeMVar (ashWake handle)) + +-- | Insert a new event, or return the already-accepted event for a replayed +-- @(source, sourceEventId)@ pair without dispatching it again. One STM +-- transaction, so two concurrent submissions of the same key can never +-- both "win". +insertOrGetExisting + :: AlarmStoreHandle + -> AlarmSource + -> UTCTime -- ^ receivedAt + -> IngressRequest + -> IO (AlarmCursor, AlarmEvent, Bool) -- ^ (cursor, event, was newly created) +insertOrGetExisting handle src receivedAt req = + atomically do + idx <- readTVar (ashIndex handle) + case Map.lookup (src, irSourceEventId req) idx of + Just cursor -> do + evs <- readTVar (ashEvents handle) + case Map.lookup cursor evs of + Just ev -> pure (cursor, ev, False) + -- Indexed but pruned already: treat as a fresh submission. This + -- can only happen if pruning ever removed an event without also + -- removing its index entry -- 'pruneOnce' takes care to avoid + -- that, but resolving to a fresh insert here is a safe fallback + -- either way. + Nothing -> insertNew + Nothing -> insertNew + where + insertNew = do + seqNum <- stateTVar (ashNextSeq handle) (\n -> (n, n + 1)) + let cursor = AlarmCursor seqNum + eid = ashStartTag handle <> "-" <> Text.pack (show seqNum) + ev = AlarmEvent + { schemaVersion = 1 + , eventId = eid + , sourceEventId = irSourceEventId req + , raisedAt = irRaisedAt req + , receivedAt = receivedAt + , source = src + , ruleId = irRuleId req + , severity = irSeverity req + , summary = irSummary req + , scope = irScope req + , labels = irLabels req + , details = irDetails req + } + modifyTVar' (ashEvents handle) (Map.insert cursor ev) + modifyTVar' (ashIndex handle) (Map.insert (src, irSourceEventId req) cursor) + pure (cursor, ev, True) + +-- | Cursor-ordered, filtered history read, strictly after the given cursor +-- (exclusive), capped at @limit@ results. +readHistory :: AlarmStoreHandle -> Maybe AlarmCursor -> Int -> AlarmFilter -> IO [(AlarmCursor, AlarmEvent)] +readHistory handle after limit filt = do + evs <- readTVarIO (ashEvents handle) + let afterOk cursor = maybe True (cursor >) after + candidates = [ (c, ev) | (c, ev) <- Map.toAscList evs, afterOk c, matchesFilter filt ev ] + pure (take (max 0 limit) candidates) + +-- | Apply the configured age\/count retention limits. Removing an event from +-- 'ashEvents' always removes its @(source, sourceEventId)@ entry from +-- 'ashIndex' in the /same/ transaction -- if these two maps were ever +-- allowed to drift, a resubmission of a since-pruned key would return a +-- cursor that no longer resolves in 'readHistory'. +pruneOnce :: AlarmStoreHandle -> UTCTime -> IO () +pruneOnce handle now = atomically do + evs <- readTVar (ashEvents handle) + let kept = applyCount (applyAge evs) + dropped = Map.difference evs kept + writeTVar (ashEvents handle) kept + modifyTVar' (ashIndex handle) (removeDropped dropped) + where + retention = ashRetention handle + + applyAge :: Map AlarmCursor AlarmEvent -> Map AlarmCursor AlarmEvent + applyAge evs = case arcMaxAgeSeconds retention of + Nothing -> evs + Just maxAgeSecs -> Map.filter (\ev -> diffUTCTime now (receivedAt ev) <= fromIntegral maxAgeSecs) evs + + applyCount :: Map AlarmCursor AlarmEvent -> Map AlarmCursor AlarmEvent + applyCount evs = case arcMaxEvents retention of + Nothing -> evs + Just maxN -> Map.fromDistinctDescList (take (fromIntegral maxN) (Map.toDescList evs)) + + removeDropped + :: Map AlarmCursor AlarmEvent + -> Map (AlarmSource, Text) AlarmCursor + -> Map (AlarmSource, Text) AlarmCursor + removeDropped dropped idx = foldr Map.delete idx keysToRemove + where + keysToRemove = [ (source ev, sourceEventId ev) | ev <- Map.elems dropped ] diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs new file mode 100644 index 00000000000..391d576df07 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TimeseriesRules.hs @@ -0,0 +1,262 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | The timeseries-query alarm producer (concept-doc § Producers — +-- Timeseries rules). Periodically evaluates a @cardano-timeseries-io@ +-- boolean query against the in-process 'TimeseriesHandle' and, per +-- output series, runs a small edge-triggered state machine: +-- +-- false/missing → pending → publish +-- ^ | | +-- `--------- '--- false ---' +-- +-- Each output series has a deterministic key derived from its labels; +-- the @sourceEventId@ embeds that key so the alarm store dedupes +-- per-series edges. When @repeatEvery@ is configured, a still-true +-- series republishes at that interval with a fresh window-index in the +-- key. +-- +-- Errors and missing data are traced as health information; they never +-- publish false alarms. Query execution is bounded by a timeout so a +-- pathological rule cannot starve normal timeseries ingestion. +module Cardano.Tracer.Handlers.Alarms.TimeseriesRules + ( TimeseriesAlarmRule + , tarRuleId + , tarQuery + , tarEvaluateEvery + , timeseriesAlarmSource + , timeseriesRuleFromConfig + , SamplePoint (..) + , decodeSamples + , evaluateOnce + , ruleRequests + , SeriesState (..) + ) where + +import Cardano.Logging.Types (SeverityS) +import Cardano.Timeseries.API (Value (..)) +import Cardano.Timeseries.Domain.Instant (Instant (..)) +import Cardano.Tracer.Configuration (AlarmsTimeseriesRuleConfig (..)) +import Cardano.Tracer.Handlers.Alarms.Types + +import Data.Aeson (object, (.=)) +import Data.IORef (IORef, atomicModifyIORef', newIORef) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import qualified Data.Set as Set +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, diffUTCTime) +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) + +-- | Runtime form of a timeseries rule. The mutable 'IORef' holds the +-- per-series state machines — inactive, pending-with-since, or +-- active-since-published. +data TimeseriesAlarmRule = TimeseriesAlarmRule + { tarRuleId :: !RuleId + , tarSummary :: !Text + , tarSeverity :: !SeverityS + , tarQuery :: !Text + , tarEvaluateEvery :: !Word64 -- ^ seconds + , tarFor :: !Word64 -- ^ seconds; 0 means "publish on first true" + , tarRepeatEvery :: !(Maybe Word64) -- ^ seconds between reminders while still true + , tarLabels :: !(Map Text Text) + , tarSeriesState :: !(IORef (Map SeriesKey SeriesState)) + } + +-- | Canonical, deterministic key derived from a sample's labels. Encoded +-- as @k1=v1,k2=v2@ with keys sorted so semantically equal maps produce +-- the same key. +newtype SeriesKey = SeriesKey { unSeriesKey :: Text } + deriving stock (Eq, Ord, Show) + +-- | Per-series state. +data SeriesState + = Inactive + -- ^ Last observed sample was false or missing. + | Pending !UTCTime + -- ^ Sample is truthy; waiting for the @for@ duration to elapse + -- before publishing. + | Active !UTCTime + -- ^ Sample stayed truthy for @for@ seconds and has been published + -- at least once; the field is @publishedAt@ (last publish time). + deriving stock (Eq, Show) + +-- | One boolean sample decoded from a query response, tagged with its +-- labels (the source of 'SeriesKey'). +data SamplePoint = SamplePoint + { spLabels :: !(Map Text Text) + , spTruth :: !Bool + } + deriving stock (Eq, Show) + +-- | The fixed trusted 'source' for alarms raised by timeseries rules. +-- Internal producer; never passes HTTP authentication. +timeseriesAlarmSource :: AlarmSource +timeseriesAlarmSource = AlarmSource "timeseries" + +-------------------------------------------------------------------------------- +-- Config → runtime +-------------------------------------------------------------------------------- + +timeseriesRuleFromConfig :: AlarmsTimeseriesRuleConfig -> IO TimeseriesAlarmRule +timeseriesRuleFromConfig AlarmsTimeseriesRuleConfig{..} = do + stateRef <- newIORef Map.empty + pure TimeseriesAlarmRule + { tarRuleId = RuleId atsRuleId + , tarSummary = fromMaybe defaultSummary atsSummary + , tarSeverity = atsSeverity + , tarQuery = atsQuery + , tarEvaluateEvery = max 1 atsEvaluateEvery + , tarFor = fromMaybe 0 atsFor + , tarRepeatEvery = atsRepeatEvery + , tarLabels = fromMaybe Map.empty atsLabels + , tarSeriesState = stateRef + } + where + defaultSummary = "Timeseries rule " <> atsRuleId <> " triggered" + +-------------------------------------------------------------------------------- +-- Series key +-------------------------------------------------------------------------------- + +-- | Deterministic canonical encoding of a label map. Sorted by key so +-- two 'Map's with the same content always produce the same key. +seriesKeyOf :: Map Text Text -> SeriesKey +seriesKeyOf labels = + SeriesKey (Text.intercalate "," [ k <> "=" <> v | (k, v) <- Map.toAscList labels ]) + +-------------------------------------------------------------------------------- +-- Evaluation → ingress requests +-------------------------------------------------------------------------------- + +-- | Apply one round of samples to the rule. Advances every touched +-- series through its state machine; returns one 'IngressRequest' per +-- series that just published (either a fresh edge or a scheduled +-- reminder). +-- +-- Series that appear as 'False' or as missing (not in the samples map) +-- transition to 'Inactive'. Series that appear as 'True' advance +-- 'Inactive → Pending' or, having sat in 'Pending' for at least +-- @tarFor@ seconds, transition to 'Active' and publish. +-- +-- Pure enough to be tested: takes @now@ as an argument, returns the new +-- 'SeriesState' map alongside the requests, and writes back to the +-- 'IORef' only in 'evaluateOnce'. +ruleRequests + :: TimeseriesAlarmRule + -> UTCTime -- ^ evaluation timestamp + -> [SamplePoint] -- ^ decoded query response + -> Map SeriesKey SeriesState -- ^ previous state + -> (Map SeriesKey SeriesState, [IngressRequest]) +ruleRequests rule@TimeseriesAlarmRule{tarFor, tarRepeatEvery} now samples prev = + let touched = Map.fromList [ (seriesKeyOf (spLabels sp), sp) | sp <- samples ] + allKeys = Set.toAscList (Map.keysSet touched <> Map.keysSet prev) + results = map + (\k -> advance k (Map.lookup k touched) (Map.findWithDefault Inactive k prev)) + allKeys + newState = Map.fromList [ (k, s) | (k, s, _) <- results ] + reqs = [ req | (_, _, Just req) <- results ] + in (newState, reqs) + where + advance :: SeriesKey + -> Maybe SamplePoint + -> SeriesState + -> (SeriesKey, SeriesState, Maybe IngressRequest) + advance key mSample state = case (state, isTrue mSample) of + (_, False) -> + (key, Inactive, Nothing) + (Inactive, True) -> + if tarFor == 0 + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Pending now, Nothing) + (Pending since, True) -> + let elapsed = diffUTCTime now since + in if realToFrac elapsed >= (fromIntegral tarFor :: Double) + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Pending since, Nothing) + (Active publishedAt, True) -> + case tarRepeatEvery of + Nothing -> (key, Active publishedAt, Nothing) + Just repeatSecs -> + let elapsed = diffUTCTime now publishedAt + in if realToFrac elapsed >= (fromIntegral repeatSecs :: Double) + then let req = buildRequest rule key mSample now + in (key, Active now, Just req) + else (key, Active publishedAt, Nothing) + + isTrue :: Maybe SamplePoint -> Bool + isTrue = maybe False spTruth + +buildRequest :: TimeseriesAlarmRule -> SeriesKey -> Maybe SamplePoint -> UTCTime -> IngressRequest +buildRequest TimeseriesAlarmRule{tarRuleId, tarSummary, tarSeverity, tarLabels} + seriesKey mSample now = + IngressRequest + { irSourceEventId = "ts:" <> unRuleId tarRuleId <> ":" <> unSeriesKey seriesKey + <> ":" <> Text.pack (show (windowIndex now)) + , irRaisedAt = now + , irRuleId = tarRuleId + , irSeverity = tarSeverity + , irSummary = tarSummary + , irScope = Map.empty + , irLabels = Map.union sampleLabels tarLabels + , irDetails = Just $ object + [ "seriesKey" .= unSeriesKey seriesKey + , "labels" .= sampleLabels + ] + } + where + sampleLabels = maybe Map.empty spLabels mSample + -- Millisecond-resolution window index so successive publishes (from + -- 'repeatEvery') always get distinct source-event-ids. + windowIndex :: UTCTime -> Integer + windowIndex t = floor (realToFrac (utcTimeToPOSIXSeconds t) * (1000 :: Double)) + +-- | Decode a query result into per-series boolean samples. +-- +-- Three top-level shapes are meaningful; anything else is a shape +-- error. Inside an instant vector two element shapes are accepted: +-- +-- * 'Truth'\/'Falsity' — produced by explicitly boolean queries such +-- as @map (\\x -> x > 200) (m now)@; +-- * 'Scalar' — produced by comparison filters such as @m now > 200@, +-- which keep only the series satisfying the relation. Presence of +-- a series in the result means the condition holds for it, exactly +-- like a PromQL alert expression, so a surviving 'Scalar' decodes +-- as true. Series filtered out are absent from the vector and are +-- treated as missing (false) by the state machine. +decodeSamples :: Value -> Maybe [SamplePoint] +decodeSamples val = case val of + Truth -> Just [SamplePoint Map.empty True] + Falsity -> Just [SamplePoint Map.empty False] + InstantVector v -> traverse decodeInstant v + _ -> Nothing + where + decodeInstant :: Instant Value -> Maybe SamplePoint + decodeInstant (Instant labels _ inner) = do + truth <- case inner of + Truth -> Just True + Falsity -> Just False + Scalar _ -> Just True + _ -> Nothing + pure (SamplePoint (Map.fromList (Set.toList labels)) truth) + +-- | 'IO' wrapper around 'ruleRequests': reads the previous per-series +-- state atomically, computes the new state, writes it back, and +-- returns the requests to submit. +evaluateOnce + :: TimeseriesAlarmRule + -> UTCTime + -> [SamplePoint] + -> IO [IngressRequest] +evaluateOnce rule now samples = + atomicModifyIORef' (tarSeriesState rule) \prev -> + let (newState, reqs) = ruleRequests rule now samples prev + in (newState, reqs) + diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs new file mode 100644 index 00000000000..544b715f0e0 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/TraceRules.hs @@ -0,0 +1,85 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | The trace-severity alarm producer: raises an alarm for every received +-- trace message whose severity is at or above a configured threshold. Like +-- "Cardano.Tracer.Handlers.Alarms.Consumers", this module holds the rule +-- type, its config translation, and the pure matching logic; the IO driving +-- lives in "Cardano.Tracer.Handlers.Alarms.Registry". +module Cardano.Tracer.Handlers.Alarms.TraceRules + ( TraceAlarmRule (..) + , traceRuleFromConfig + , traceAlarmSource + , traceRuleRequest + ) where + +import Cardano.Logging.Types (SeverityS, TraceObject (..)) +import Cardano.Tracer.Configuration (AlarmsTraceRuleConfig (..), alarmSeverityToText) +import Cardano.Tracer.Handlers.Alarms.Types + +import Data.Aeson (object, (.=)) +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) + +data TraceAlarmRule = TraceAlarmRule + { tarRuleId :: !RuleId + , tarSummary :: !Text + , tarThreshold :: !SeverityS + , tarSuppressSecs :: !Word64 + , tarLabels :: !(Map Text Text) + } + +traceRuleFromConfig :: AlarmsTraceRuleConfig -> TraceAlarmRule +traceRuleFromConfig AlarmsTraceRuleConfig{..} = TraceAlarmRule + { tarRuleId = RuleId atrRuleId + , tarSummary = fromMaybe defaultSummary atrSummary + , tarThreshold = atrThreshold + , tarSuppressSecs = max 1 (fromMaybe 300 atrSuppressForSecs) + , tarLabels = fromMaybe Map.empty atrLabels + } + where + defaultSummary = + "Trace message with severity at or above " + <> alarmSeverityToText atrThreshold <> " received" + +-- | The fixed trusted source identity for alarms raised by trace severity +-- rules. Internal producers never pass HTTP authentication, so the source +-- is a constant here, mirroring how external producers get their source +-- from their credential rather than choosing it themselves. +traceAlarmSource :: AlarmSource +traceAlarmSource = AlarmSource "trace" + +-- | Pure mapping from one received trace message to an alarm submission, or +-- 'Nothing' when the message is below the rule's threshold. The +-- @sourceEventId@ encodes a time window derived from the message's own +-- timestamp, so all matches from the same node and namespace within one +-- window share the store's idempotency key and collapse into one alarm -- +-- flood prevention without any extra state. +traceRuleRequest :: TraceAlarmRule -> Text -> TraceObject -> Maybe IngressRequest +traceRuleRequest TraceAlarmRule{..} nodeName trObj + | toSeverity trObj < tarThreshold = Nothing + | otherwise = Just IngressRequest + { irSourceEventId = Text.intercalate ":" + [unRuleId tarRuleId, nodeName, namespace, Text.pack (show windowIndex)] + , irRaisedAt = toTimestamp trObj + , irRuleId = tarRuleId + , irSeverity = toSeverity trObj + , irSummary = tarSummary + , irScope = Map.singleton "nodeId" nodeName + , irLabels = Map.insert "namespace" namespace tarLabels + , irDetails = Just $ object + [ "message" .= toMachine trObj + , "hostname" .= toHostname trObj + , "threadId" .= toThreadId trObj + ] + } + where + namespace = Text.intercalate "." (toNamespace trObj) + + windowIndex :: Word64 + windowIndex = floor (utcTimeToPOSIXSeconds (toTimestamp trObj)) `div` tarSuppressSecs diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs new file mode 100644 index 00000000000..7c069b44203 --- /dev/null +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Alarms/Types.hs @@ -0,0 +1,204 @@ +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | Pure domain types for the alarm subsystem (see +-- @cardano-tracer/docs/alarm-system-concept.md@). Deliberately depends only +-- on "Cardano.Tracer.Configuration"/"Cardano.Logging" -- never on +-- "Cardano.Tracer.Environment" -- mirroring how @Cardano.Timeseries.Component@ +-- is referenced by @Environment.hs@ but never references it back. +module Cardano.Tracer.Handlers.Alarms.Types + ( AlarmSource (..) + , RuleId (..) + , AlarmCursor (..) + , AlarmId + , IngressRequest (..) + , AlarmEvent (..) + , AlarmFilter (..) + , emptyAlarmFilter + , filterFromConfig + , matchesFilter + , filterNarrows + ) where + +import Cardano.Logging.Types (SeverityS) +import Cardano.Tracer.Configuration (AlarmFilterConfig (..), alarmSeverityToText, + parseAlarmSeverityText) + +import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.:?), (.=)) +import qualified Data.Aeson as Aeson +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime) +import Data.Word (Word64) + +-- | The trusted identity of an alarm producer, derived from the producer +-- credential that authenticated the ingress request. Never taken from the +-- request body itself. +newtype AlarmSource = AlarmSource { unAlarmSource :: Text } + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON) + +newtype RuleId = RuleId { unRuleId :: Text } + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON) + +-- | A monotonic sequence number assigned by the store on insertion. Serves +-- as the pagination cursor for history reads. +newtype AlarmCursor = AlarmCursor Word64 + deriving stock (Eq, Ord, Show) + deriving newtype (FromJSON, ToJSON, Enum, Num) + +-- | The server-assigned event identifier included in the wire envelope. +type AlarmId = Text + +-- | The producer-submitted payload, once JSON-parsed. Only ever reads +-- producer-owned fields: @source@, @eventId@, and @receivedAt@ in the +-- design doc's envelope are assigned by the server, so a caller-supplied +-- value for any of them in the request body is simply not read here. +data IngressRequest = IngressRequest + { irSourceEventId :: !Text + , irRaisedAt :: !UTCTime + , irRuleId :: !RuleId + , irSeverity :: !SeverityS + , irSummary :: !Text + , irScope :: !(Map Text Text) + , irLabels :: !(Map Text Text) + , irDetails :: !(Maybe Aeson.Value) + } + deriving stock (Eq, Show) + +instance FromJSON IngressRequest where + parseJSON = withObject "IngressRequest" \o -> do + irSourceEventId <- o .: "sourceEventId" + irRaisedAt <- o .: "raisedAt" + irRuleId <- RuleId <$> o .: "ruleId" + severityText <- o .: "severity" + irSeverity <- parseSeverityOrFail severityText + irSummary <- o .: "summary" + irScope <- fromMaybe Map.empty <$> o .:? "scope" + irLabels <- fromMaybe Map.empty <$> o .:? "labels" + irDetails <- o .:? "details" + pure IngressRequest{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +-- | The immutable, versioned alarm envelope. @eventId@\/@receivedAt@\/@source@ +-- are assigned by the server; everything else is copied from the +-- producer's 'IngressRequest'. +data AlarmEvent = AlarmEvent + { schemaVersion :: !Int + , eventId :: !AlarmId + , sourceEventId :: !Text + , raisedAt :: !UTCTime + , receivedAt :: !UTCTime + , source :: !AlarmSource + , ruleId :: !RuleId + , severity :: !SeverityS + , summary :: !Text + , scope :: !(Map Text Text) + , labels :: !(Map Text Text) + , details :: !(Maybe Aeson.Value) + } + deriving stock (Eq, Show) + +instance ToJSON AlarmEvent where + toJSON AlarmEvent{..} = object + [ "schemaVersion" .= schemaVersion + , "eventId" .= eventId + , "sourceEventId" .= sourceEventId + , "raisedAt" .= raisedAt + , "receivedAt" .= receivedAt + , "source" .= unAlarmSource source + , "ruleId" .= unRuleId ruleId + , "severity" .= alarmSeverityToText severity + , "summary" .= summary + , "scope" .= scope + , "labels" .= labels + , "details" .= details + ] + +instance FromJSON AlarmEvent where + parseJSON = withObject "AlarmEvent" \o -> do + schemaVersion <- o .: "schemaVersion" + eventId <- o .: "eventId" + sourceEventId <- o .: "sourceEventId" + raisedAt <- o .: "raisedAt" + receivedAt <- o .: "receivedAt" + source <- AlarmSource <$> o .: "source" + ruleId <- RuleId <$> o .: "ruleId" + severityText <- o .: "severity" + severity <- parseSeverityOrFail severityText + summary <- o .: "summary" + scope <- fromMaybe Map.empty <$> o .:? "scope" + labels <- fromMaybe Map.empty <$> o .:? "labels" + details <- o .:? "details" + pure AlarmEvent{..} + where + parseSeverityOrFail t = + maybe (fail ("unknown severity: " <> Text.unpack t)) pure (parseAlarmSeverityText t) + +-- | A conjunctive selection over 'AlarmEvent's, used both for a static +-- consumer's configuration and for a reader credential's allowed +-- (ceiling) or requested filter. Scope\/label matching requires +-- the event to carry every key\/value pair listed here (a submap check), +-- not just an intersection. +data AlarmFilter = AlarmFilter + { afSource :: !(Maybe AlarmSource) + , afRuleId :: !(Maybe RuleId) + , afMinSeverity :: !(Maybe SeverityS) + , afScope :: !(Map Text Text) + , afLabels :: !(Map Text Text) + } + deriving stock (Eq, Show) + +emptyAlarmFilter :: AlarmFilter +emptyAlarmFilter = AlarmFilter Nothing Nothing Nothing Map.empty Map.empty + +filterFromConfig :: AlarmFilterConfig -> AlarmFilter +filterFromConfig AlarmFilterConfig{..} = AlarmFilter + { afSource = AlarmSource <$> afcSource + , afRuleId = RuleId <$> afcRuleId + , afMinSeverity = afcMinSeverity + , afScope = fromMaybe Map.empty afcScope + , afLabels = fromMaybe Map.empty afcLabels + } + +isSubmapOf :: (Ord k, Eq v) => Map k v -> Map k v -> Bool +isSubmapOf = Map.isSubmapOfBy (==) + +matchesFilter :: AlarmFilter -> AlarmEvent -> Bool +matchesFilter AlarmFilter{..} ev = + maybe True (== source ev) afSource + && maybe True (== ruleId ev) afRuleId + && maybe True (<= severity ev) afMinSeverity + && afScope `isSubmapOf` scope ev + && afLabels `isSubmapOf` labels ev + +-- | Is @requested@ at least as strict as @ceiling@ -- i.e. can it only narrow, +-- never broaden, what the credential allows? Used to reject a reader's +-- requested history filter that would see more than their credential +-- permits. +filterNarrows :: AlarmFilter -> AlarmFilter -> Bool +filterNarrows ceilingFilter requestedFilter = + narrowsEq (afSource ceilingFilter) (afSource requestedFilter) + && narrowsEq (afRuleId ceilingFilter) (afRuleId requestedFilter) + && narrowsSeverity (afMinSeverity ceilingFilter) (afMinSeverity requestedFilter) + && afScope ceilingFilter `isSubmapOf` afScope requestedFilter + && afLabels ceilingFilter `isSubmapOf` afLabels requestedFilter + where + -- A single-value equality constraint can only be kept as-is, never + -- dropped or changed, once the ceiling fixes it. + narrowsEq :: Eq a => Maybe a -> Maybe a -> Bool + narrowsEq Nothing _ = True + narrowsEq (Just c) (Just r) = c == r + narrowsEq (Just _) Nothing = False + + narrowsSeverity :: Maybe SeverityS -> Maybe SeverityS -> Bool + narrowsSeverity Nothing _ = True + narrowsSeverity (Just c) (Just r) = r >= c + narrowsSeverity (Just _) Nothing = False diff --git a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs index 003c938e6b8..646e000044d 100644 --- a/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs +++ b/cardano-tracer/src/Cardano/Tracer/Handlers/Logs/TraceObjects.hs @@ -10,12 +10,14 @@ module Cardano.Tracer.Handlers.Logs.TraceObjects import Cardano.Logging (TraceObject) import Cardano.Tracer.Configuration import Cardano.Tracer.Environment +import Cardano.Tracer.Handlers.Alarms.Registry (checkTraceObjectsForAlarms) import Cardano.Tracer.Handlers.Logs.File import Cardano.Tracer.Handlers.Logs.Journal import Cardano.Tracer.Types import Cardano.Tracer.Utils import Control.Concurrent.Async (forConcurrently_) +import Data.Foldable (for_) import qualified Data.Map as Map import System.IO (Handle, hClose) @@ -37,6 +39,8 @@ traceObjectsHandler tracerEnv nodeId traceObjects = do loggingParams nodeName teCurrentLogLock traceObjects JournalMode -> writeTraceObjectsToJournal logFormat nodeName traceObjects + for_ teAlarmRegistry \alarmRegistry -> + checkTraceObjectsForAlarms alarmRegistry nodeName traceObjects teReforwardTraceObjects traceObjects where TracerEnv { teConfig = TracerConfig{ logging, verbosity } @@ -44,6 +48,7 @@ traceObjectsHandler tracerEnv nodeId traceObjects = do , teReforwardTraceObjects , teRegistry , teTracer + , teAlarmRegistry } = tracerEnv deregisterNodeId :: TracerEnv -> NodeId -> IO () diff --git a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs index 151e5e365a9..d0dab1076f7 100644 --- a/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs +++ b/cardano-tracer/src/Cardano/Tracer/MetaTrace.hs @@ -84,6 +84,36 @@ data TracerTrace { ttConnection :: HowToConnect , ttMessage :: String } + | TracerStartedAlarms + { ttAlarmsEndpoint :: Endpoint + } + | TracerAlarmAccepted + { ttAlarmAcceptedSource :: Text + , ttAlarmAcceptedRuleId :: Text + , ttAlarmAcceptedSeverity :: SeverityS + } + | TracerAlarmDuplicate + { ttAlarmDuplicateSource :: Text + , ttAlarmDuplicateSourceEventId :: Text + } + | TracerAlarmRejected + { ttAlarmRejectedReason :: Text + } + | TracerAlarmDispatched + { ttAlarmDispatchedConsumer :: Text + , ttAlarmDispatchedSource :: Text + , ttAlarmDispatchedRuleId :: Text + , ttAlarmDispatchedSeverity :: SeverityS + , ttAlarmDispatchedSummary :: Text + } + | TracerAlarmHistoryRead + { ttAlarmHistoryReader :: Text + , ttAlarmHistoryResultCount :: Int + } + | TracerAlarmTimeseriesEvalFailed + { ttAlarmTimeseriesEvalFailedRule :: Text + , ttAlarmTimeseriesEvalFailedReason :: Text + } deriving Show -- | A bundle of domain-split tracers used in the application. @@ -184,6 +214,43 @@ instance LogFormatting TracerTrace where , "conn" .= ttConnection , "message" .= ttMessage ] + TracerStartedAlarms{..} -> mconcat + [ "kind" .= AE.String "TracerStartedAlarms" + , "endpoint" .= ttAlarmsEndpoint + ] + TracerAlarmAccepted{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmAccepted" + , "source" .= ttAlarmAcceptedSource + , "ruleId" .= ttAlarmAcceptedRuleId + , "severity" .= ttAlarmAcceptedSeverity + ] + TracerAlarmDuplicate{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmDuplicate" + , "source" .= ttAlarmDuplicateSource + , "sourceEventId" .= ttAlarmDuplicateSourceEventId + ] + TracerAlarmRejected{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmRejected" + , "reason" .= ttAlarmRejectedReason + ] + TracerAlarmDispatched{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmDispatched" + , "consumer" .= ttAlarmDispatchedConsumer + , "source" .= ttAlarmDispatchedSource + , "ruleId" .= ttAlarmDispatchedRuleId + , "severity" .= ttAlarmDispatchedSeverity + , "summary" .= ttAlarmDispatchedSummary + ] + TracerAlarmHistoryRead{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmHistoryRead" + , "reader" .= ttAlarmHistoryReader + , "resultCount" .= ttAlarmHistoryResultCount + ] + TracerAlarmTimeseriesEvalFailed{..} -> mconcat + [ "kind" .= AE.String "TracerAlarmTimeseriesEvalFailed" + , "rule" .= ttAlarmTimeseriesEvalFailedRule + , "reason" .= ttAlarmTimeseriesEvalFailedReason + ] instance MetaTrace TracerTrace where namespaceFor TracerBuildInfo {} = Namespace [] ["BuildInfo"] @@ -208,6 +275,13 @@ instance MetaTrace TracerTrace where namespaceFor TracerError {} = Namespace [] ["Error"] namespaceFor TracerResource {} = Namespace [] ["Resources"] namespaceFor TracerForwardingInterrupted {} = Namespace [] ["ForwardingInterrupted"] + namespaceFor TracerStartedAlarms {} = Namespace [] ["StartedAlarms"] + namespaceFor TracerAlarmAccepted {} = Namespace [] ["AlarmAccepted"] + namespaceFor TracerAlarmDuplicate {} = Namespace [] ["AlarmDuplicate"] + namespaceFor TracerAlarmRejected {} = Namespace [] ["AlarmRejected"] + namespaceFor TracerAlarmDispatched {} = Namespace [] ["AlarmDispatched"] + namespaceFor TracerAlarmHistoryRead {} = Namespace [] ["AlarmHistoryRead"] + namespaceFor TracerAlarmTimeseriesEvalFailed {} = Namespace [] ["AlarmTimeseriesEvalFailed"] severityFor (Namespace _ ["BuildInfo"]) _ = Just Info severityFor (Namespace _ ["ParamsAre"]) _ = Just Warning @@ -231,6 +305,13 @@ instance MetaTrace TracerTrace where severityFor (Namespace _ ["Error"]) _ = Just Error severityFor (Namespace _ ["Resources"]) _ = Just Info severityFor (Namespace _ ["ForwardingInterrupted"]) _ = Just Warning + severityFor (Namespace _ ["StartedAlarms"]) _ = Just Info + severityFor (Namespace _ ["AlarmAccepted"]) _ = Just Info + severityFor (Namespace _ ["AlarmDuplicate"]) _ = Just Debug + severityFor (Namespace _ ["AlarmRejected"]) _ = Just Warning + severityFor (Namespace _ ["AlarmDispatched"]) _ = Just Info + severityFor (Namespace _ ["AlarmHistoryRead"]) _ = Just Info + severityFor (Namespace _ ["AlarmTimeseriesEvalFailed"]) _ = Just Warning severityFor _ _ = Nothing documentFor _ = Just "" @@ -258,6 +339,13 @@ instance MetaTrace TracerTrace where , Namespace [] ["Error"] , Namespace [] ["Resources"] , Namespace [] ["ForwardingInterrupted"] + , Namespace [] ["StartedAlarms"] + , Namespace [] ["AlarmAccepted"] + , Namespace [] ["AlarmDuplicate"] + , Namespace [] ["AlarmRejected"] + , Namespace [] ["AlarmDispatched"] + , Namespace [] ["AlarmHistoryRead"] + , Namespace [] ["AlarmTimeseriesEvalFailed"] ] stderrShowTracer :: Show a => Trace IO a diff --git a/cardano-tracer/src/Cardano/Tracer/Run.hs b/cardano-tracer/src/Cardano/Tracer/Run.hs index bd605acec7e..d894f2cc3e7 100644 --- a/cardano-tracer/src/Cardano/Tracer/Run.hs +++ b/cardano-tracer/src/Cardano/Tracer/Run.hs @@ -16,6 +16,8 @@ import Cardano.Tracer.Acceptors.Run import Cardano.Tracer.CLI import Cardano.Tracer.Configuration import Cardano.Tracer.Environment +import qualified Cardano.Tracer.Handlers.Alarms.Registry as Alarms +import Cardano.Tracer.Handlers.Alarms.Server (runAlarms) import Cardano.Tracer.Handlers.Logs.Rotator import Cardano.Tracer.Handlers.Metrics.Servers import Cardano.Tracer.Handlers.ReForwarder @@ -92,6 +94,10 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do registry <- newRegistry !timeseriesHandle <- for (hasTimeseries config) (const $ Timeseries.create @(Tree _) tr.timeseries Nothing) + !alarmRegistry <- for (alarms config) (Alarms.newAlarmRegistry tr.assorted) + case (alarmRegistry, timeseriesHandle) of + (Just reg, Just ts) -> Alarms.runTimeseriesEvaluator reg ts + _ -> pure () -- Environment for all following functions. let tracerEnv :: TracerEnv @@ -110,6 +116,7 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do , teStateDir = stateDir , teMetricsHelp = mHelp , teTimeseriesHandle = timeseriesHandle + , teAlarmRegistry = alarmRegistry } -- Specify what should be done before 'cardano-tracer' stops. @@ -123,6 +130,7 @@ doRunCardanoTracer config stateDir tr protocolsBrake dpRequestors = do [ runLogsRotator tracerEnv , runMetricsServers tracerEnv , runAcceptors tracerEnv + , runAlarms tracerEnv ] -- NB. this fails silently if there's any read or decode error when an external JSON file is provided diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs index 6b4907730c3..5d74a6c2e41 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Acceptor.hs @@ -63,6 +63,7 @@ launchAcceptorsSimple mode localSock dpName = do , teStateDir = Nothing , teMetricsHelp = [] , teTimeseriesHandle = Nothing + , teAlarmRegistry = Nothing } -- NOTE: no reforwarding in this acceptor. @@ -81,6 +82,7 @@ launchAcceptorsSimple mode localSock dpName = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams "/tmp/demo-acceptor" FileMode ForHuman] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs new file mode 100644 index 00000000000..d62bcfc38a9 --- /dev/null +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/Tests.hs @@ -0,0 +1,434 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} + +-- | Real property tests for the alarm subsystem. Each property targets a +-- concrete promise from @cardano-tracer/docs/alarm-system-concept.md@: +-- idempotency, cursor ordering, retention, filter algebra, reader +-- ceiling, and trace-rule flood suppression. Fixture-style tests (a +-- single hand-picked input) are kept only where the behaviour is truly +-- singleton — everything else is generator-driven. +module Cardano.Tracer.Test.Alarms.Tests + ( tests + ) where + +import Cardano.Logging (DetailLevel (..), SeverityF (..), SeverityS (..), + TraceObject (..)) +import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Store +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) + +import Control.Concurrent.Async (forConcurrently) +import Data.Aeson (decode, encode) +import qualified Data.List as List +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as Map +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime) +import Data.Time.Clock.POSIX (posixSecondsToUTCTime, utcTimeToPOSIXSeconds) + +import Test.Tasty +import Test.Tasty.QuickCheck hiding (labels) + +-------------------------------------------------------------------------------- +-- Test suite +-------------------------------------------------------------------------------- + +tests :: TestTree +tests = testGroup "Test.Alarms" + [ testGroup "envelope" + [ testProperty "severity vocabulary lowercase round-trip" propSeverityRoundTrip + , testProperty "AlarmEvent JSON round-trip" propAlarmEventRoundTrip + ] + , testGroup "store" + [ testProperty "idempotent double-submit returns same eventId" propIdempotentSubmit + , testProperty "concurrent submit creates exactly one" propConcurrentSubmit + , testProperty "prune then resubmit issues a fresh eventId" propPruneThenResubmit + , testProperty "cursor is strictly monotonic across inserts" propCursorMonotonic + , testProperty "retention maxEvents keeps exactly maxN" propRetentionMaxEvents + , testProperty "retention maxAgeSeconds drops older events" propRetentionMaxAge + , testProperty "readHistory after=cursor is exclusive" propHistoryAfterExclusive + ] + , testGroup "filter algebra" + [ testProperty "empty filter accepts every event" propEmptyFilterAccepts + , testProperty "scope is submap, not intersection" propScopeIsSubmap + , testProperty "minSeverity is <= (inclusive)" propMinSeverityInclusive + , testProperty "filterNarrows is reflexive" propFilterNarrowsReflexive + , testProperty "filterNarrows is transitive" propFilterNarrowsTransitive + , testProperty "broader-than-ceiling filter is rejected" propBroaderRejected + ] + , testGroup "trace rules" + [ testProperty "rule fires only at or above threshold" propTraceRuleThreshold + , testProperty "matches in one window collapse to one alarm" propTraceRuleSuppression + , testProperty "matches in a new window raise a fresh alarm" propTraceRuleNewWindow + ] + ] + +-------------------------------------------------------------------------------- +-- Generators +-------------------------------------------------------------------------------- + +genSeverity :: Gen SeverityS +genSeverity = elements [Debug, Info, Notice, Warning, Error, Critical, Alert, Emergency] + +genShortText :: Gen Text +genShortText = do + n <- chooseInt (1, 16) + Text.pack <$> vectorOf n (elements (['a'..'z'] <> ['0'..'9'] <> "_-")) + +genSource :: Gen AlarmSource +genSource = AlarmSource <$> elements ["trace", "hermod-recon", "timeseries", "test-src"] + +genRule :: Gen RuleId +genRule = RuleId <$> elements ["rule-a", "rule-b", "rule-c", "rule-d"] + +genLabels :: Gen (Map Text Text) +genLabels = do + n <- chooseInt (0, 3) + Map.fromList <$> vectorOf n ((,) <$> elements ["team", "site", "region", "shard"] + <*> elements ["ops", "dev", "eu", "us", "0", "1"]) + +genUTC :: Gen UTCTime +genUTC = do + s <- chooseInteger (1_700_000_000, 1_900_000_000) + pure (posixSecondsToUTCTime (fromIntegral s)) + +genIngress :: Gen IngressRequest +genIngress = IngressRequest + <$> genShortText + <*> genUTC + <*> genRule + <*> genSeverity + <*> genShortText + <*> genLabels + <*> genLabels + <*> pure Nothing + +genEvent :: Gen AlarmEvent +genEvent = do + ir <- genIngress + src <- genSource + now <- genUTC + eid <- genShortText + pure AlarmEvent + { schemaVersion = 1 + , eventId = eid + , sourceEventId = irSourceEventId ir + , raisedAt = irRaisedAt ir + , receivedAt = now + , source = src + , ruleId = irRuleId ir + , severity = irSeverity ir + , summary = irSummary ir + , scope = irScope ir + , labels = irLabels ir + , details = irDetails ir + } + +genFilter :: Gen AlarmFilter +genFilter = AlarmFilter + <$> oneof [pure Nothing, Just <$> genSource] + <*> oneof [pure Nothing, Just <$> genRule] + <*> oneof [pure Nothing, Just <$> genSeverity] + <*> genLabels + <*> genLabels + +-------------------------------------------------------------------------------- +-- Envelope properties +-------------------------------------------------------------------------------- + +propSeverityRoundTrip :: Property +propSeverityRoundTrip = forAll genSeverity \sev -> + parseAlarmSeverityText (alarmSeverityToText sev) === Just sev + +propAlarmEventRoundTrip :: Property +propAlarmEventRoundTrip = forAll genEvent \ev -> + decode (encode ev) === Just ev + +-------------------------------------------------------------------------------- +-- Store properties +-------------------------------------------------------------------------------- + +testSource :: AlarmSource +testSource = AlarmSource "hermod-recon" + +mkRequestFromParts :: Text -> SeverityS -> IO IngressRequest +mkRequestFromParts sid sev = do + now <- getCurrentTime + pure IngressRequest + { irSourceEventId = sid + , irRaisedAt = now + , irRuleId = RuleId "test-rule" + , irSeverity = sev + , irSummary = "test summary" + , irScope = Map.empty + , irLabels = Map.empty + , irDetails = Nothing + } + +propIdempotentSubmit :: Property +propIdempotentSubmit = forAll genShortText \sid -> forAll genSeverity \sev -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + (_, ev1, created1) <- insertOrGetExisting store testSource now req + (_, ev2, created2) <- insertOrGetExisting store testSource now req + pure $ conjoin + [ counterexample "first submit must create" (property created1) + , counterexample "second submit must not create" (property (not created2)) + , counterexample "eventIds must match" (eventId ev1 === eventId ev2) + ] + +propConcurrentSubmit :: Property +propConcurrentSubmit = forAll genShortText \sid -> forAll genSeverity \sev -> + forAll (chooseInt (2, 32)) \fanout -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + results <- forConcurrently [1 .. fanout] \_ -> + insertOrGetExisting store testSource now req + let createdCount = length (filter (\(_, _, c) -> c) results) + pure (createdCount === 1) + +propPruneThenResubmit :: Property +propPruneThenResubmit = forAll genShortText \sid -> forAll genSeverity \sev -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig (Just 0) Nothing) + req <- mkRequestFromParts sid sev + now <- getCurrentTime + (_, ev1, _) <- insertOrGetExisting store testSource now req + later <- getCurrentTime + pruneOnce store later + (_, ev2, created2) <- insertOrGetExisting store testSource later req + pure $ conjoin + [ counterexample "resubmission after prune must create anew" (property created2) + , counterexample "eventIds must differ" (eventId ev1 =/= eventId ev2) + ] + +propCursorMonotonic :: Property +propCursorMonotonic = forAll (chooseInt (2, 16)) \n -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + now <- getCurrentTime + cursors <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + (cursor, _, _) <- insertOrGetExisting store testSource now req + pure cursor) [1 .. n] + pure (cursors === List.sort cursors .&&. length cursors === length (List.nub cursors)) + +-- | Insert N events; configure retention to keep exactly K < N; after +-- 'pruneOnce', exactly K events remain. +propRetentionMaxEvents :: Property +propRetentionMaxEvents = + forAll (chooseInt (4, 12)) \n -> + forAll (chooseInt (1, n - 1)) \k -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing (Just (fromIntegral k))) + now <- getCurrentTime + _ <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + insertOrGetExisting store testSource now req) [1 .. n] + pruneOnce store now + kept <- readHistory store Nothing 100 emptyAlarmFilter + pure (length kept === k) + +-- | An event received long ago is dropped by a maxAgeSeconds=1 prune. +propRetentionMaxAge :: Property +propRetentionMaxAge = forAll genShortText \sid -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig (Just 1) Nothing) + now <- getCurrentTime + let long_ago = addUTCTime (negate 3600) now + req <- mkRequestFromParts sid Warning + _ <- insertOrGetExisting store testSource long_ago req + pruneOnce store now + kept <- readHistory store Nothing 100 emptyAlarmFilter + pure (length kept === 0) + +-- | 'readHistory's 'after' parameter is exclusive: every returned +-- cursor is strictly greater than 'after'. +propHistoryAfterExclusive :: Property +propHistoryAfterExclusive = forAll (chooseInt (2, 8)) \n -> ioProperty do + store <- newAlarmStore (AlarmsRetentionConfig Nothing Nothing) + now <- getCurrentTime + _ <- traverse (\i -> do + req <- mkRequestFromParts (Text.pack ("evt-" <> show i)) Warning + insertOrGetExisting store testSource now req) [1 .. n] + all_ <- readHistory store Nothing 100 emptyAlarmFilter + case all_ of + [] -> pure (property False) + ((c, _) : _) -> do + rest <- readHistory store (Just c) 100 emptyAlarmFilter + pure $ counterexample ("first cursor " <> show c <> " leaked into after-read") + (all ((> c) . fst) rest) + +-------------------------------------------------------------------------------- +-- Filter algebra properties +-------------------------------------------------------------------------------- + +-- | The empty filter accepts every event unconditionally. +propEmptyFilterAccepts :: Property +propEmptyFilterAccepts = forAll genEvent \ev -> + property (matchesFilter emptyAlarmFilter ev) + +-- | Concept doc: "requires the event to carry every key/value pair +-- listed here (a submap check), not just an intersection." Test: if +-- the filter demands a key the event does not have, the match must +-- fail even when the event has other scope entries. +propScopeIsSubmap :: Property +propScopeIsSubmap = + forAll genEvent \ev -> + forAll genLabels \extraScope -> + let evWithScope = ev { scope = Map.union (scope ev) extraScope } + filt = emptyAlarmFilter { afScope = Map.insert "not-in-event" "x" (scope evWithScope) } + in property (not (matchesFilter filt evWithScope)) + +-- | 'minSeverity' is inclusive: an event whose severity equals the +-- filter's minimum matches. +propMinSeverityInclusive :: Property +propMinSeverityInclusive = forAll genEvent \ev -> + let filt = emptyAlarmFilter { afMinSeverity = Just (severity ev) } + in property (matchesFilter filt ev) + +-- | Any filter narrows itself. Reflexivity is the base case of the +-- ceiling algebra: without it a reader cannot even keep its own +-- credential's filter. +propFilterNarrowsReflexive :: Property +propFilterNarrowsReflexive = forAll genFilter \f -> + property (filterNarrows f f) + +-- | Generate a filter that narrows the given ceiling: fixed single-value +-- constraints are kept verbatim, open ones may become fixed, the minimum +-- severity may only rise, and the scope/label maps may only gain entries. +genNarrowing :: AlarmFilter -> Gen AlarmFilter +genNarrowing ceiling_ = do + src <- keepOrFix (afSource ceiling_) genSource + rid <- keepOrFix (afRuleId ceiling_) genRule + sev <- case afMinSeverity ceiling_ of + Nothing -> oneof [pure Nothing, Just <$> genSeverity] + Just s -> Just <$> genSeverity `suchThat` (>= s) + -- left-biased union: every ceiling entry survives with its value + scope <- Map.union (afScope ceiling_) <$> genLabels + lbls <- Map.union (afLabels ceiling_) <$> genLabels + pure (AlarmFilter src rid sev scope lbls) + where + keepOrFix :: Maybe a -> Gen a -> Gen (Maybe a) + keepOrFix (Just c) _ = pure (Just c) + keepOrFix Nothing gen = oneof [pure Nothing, Just <$> gen] + +-- | If 'b' narrows 'a' and 'c' narrows 'b', then 'c' narrows 'a'. +-- Guarantees the ceiling relation composes across chained checks. The +-- chain is generated by construction ('genNarrowing') — random +-- independent filters almost never narrow each other, so an '==>' +-- precondition would discard virtually every case. +propFilterNarrowsTransitive :: Property +propFilterNarrowsTransitive = + forAll genFilter \a -> + forAll (genNarrowing a) \b -> + forAll (genNarrowing b) \c -> + counterexample ("a=" <> show a <> " b=" <> show b <> " c=" <> show c) $ + conjoin + [ counterexample "generator must narrow: b vs a" (filterNarrows a b) + , counterexample "generator must narrow: c vs b" (filterNarrows b c) + , counterexample "transitivity violated: c vs a" (filterNarrows a c) + ] + +-- | Broadening is rejected: given a ceiling that fixes 'source', a +-- requested filter that drops the source field or picks a different +-- value is not permitted. +propBroaderRejected :: Property +propBroaderRejected = + let ceiling_ = emptyAlarmFilter { afSource = Just (AlarmSource "trace") } + dropped = emptyAlarmFilter -- afSource = Nothing broadens the ceiling + differing = emptyAlarmFilter { afSource = Just (AlarmSource "somewhere-else") } + in conjoin + [ counterexample "dropping 'source' must be rejected" (not (filterNarrows ceiling_ dropped)) + , counterexample "differing 'source' value must be rejected" (not (filterNarrows ceiling_ differing)) + ] + +-------------------------------------------------------------------------------- +-- Trace-rule properties +-------------------------------------------------------------------------------- + +-- | One trace rule at Error threshold, 60s suppression window. +testAlarmsConfig :: AlarmsConfig +testAlarmsConfig = AlarmsConfig + { alEndpoint = Endpoint "127.0.0.1" 0 Nothing + , alAllowInsecure = Just True + , alRetention = Nothing + , alLimits = Nothing + , alAuthentication = AlarmsAuthConfig [] [] + , alConsumers = [] + , alTraceRules = Just + [ AlarmsTraceRuleConfig + { atrRuleId = "error-traces" + , atrSummary = Nothing + , atrThreshold = Error + , atrSuppressForSecs = Just 60 + , atrLabels = Nothing + } + ] + , alTimeseriesRules = Nothing + } + +newTestRegistry :: IO AlarmRegistry +newTestRegistry = do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + newAlarmRegistry (assorted bundle) testAlarmsConfig + +mkTraceObject :: SeverityS -> UTCTime -> TraceObject +mkTraceObject sev at = TraceObject + { toHuman = Nothing + , toMachine = "{\"msg\":\"test\"}" + , toNamespace = ["Test", "Alarm"] + , toSeverity = sev + , toDetails = DNormal + , toTimestamp = at + , toHostname = "localhost" + , toThreadId = "1" + } + +-- | Round a UTCTime down to the nearest 'window'-second boundary. +snapToWindow :: Integer -> UTCTime -> UTCTime +snapToWindow window t = + let epoch = floor (utcTimeToPOSIXSeconds t) :: Integer + snap = (epoch `div` window) * window + in posixSecondsToUTCTime (fromIntegral snap) + +-- | For any severity, the rule fires iff sev >= threshold. +propTraceRuleThreshold :: Property +propTraceRuleThreshold = forAll genSeverity \sev -> ioProperty do + registry <- newTestRegistry + now <- getCurrentTime + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject sev now] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + let expected = if sev >= Error then 1 else 0 + pure (length events === expected) + +-- | Two matches whose timestamps land in the same 60s window collapse +-- into one alarm regardless of their severities. Both timestamps are +-- taken relative to a window-aligned boundary so we can guarantee they +-- land in the same window index. +propTraceRuleSuppression :: Property +propTraceRuleSuppression = ioProperty $ do + registry <- newTestRegistry + now <- getCurrentTime + let boundary = snapToWindow 60 now + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error boundary] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Critical (addUTCTime 30 boundary)] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 1) + +-- | A match one full window after another raises a fresh alarm. The +-- second timestamp is +60s from a window boundary, guaranteed to land +-- in the next window index. +propTraceRuleNewWindow :: Property +propTraceRuleNewWindow = ioProperty $ do + registry <- newTestRegistry + now <- getCurrentTime + let boundary = snapToWindow 60 now + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error boundary] + checkTraceObjectsForAlarms registry "node-1" [mkTraceObject Error (addUTCTime 60 boundary)] + events <- readHistoryFiltered registry Nothing 10 emptyAlarmFilter + pure (length events === 2) diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs new file mode 100644 index 00000000000..d283d263914 --- /dev/null +++ b/cardano-tracer/test/Cardano/Tracer/Test/Alarms/TimeseriesTests.hs @@ -0,0 +1,286 @@ +{-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | Example test cases for timeseries alarm rules: three rules ported +-- from the Grafana catalogue in +-- @cardano-tracer/docs/grafana-alerts-as-timeseries-rules.md@, each +-- driven once with data that must NOT raise an alarm and once with data +-- that MUST raise exactly one. +-- +-- These are Level-1 tests in the sense of +-- @cardano-tracer/docs/timeseries-alarm-testing.md@: no server and no +-- forwarder — samples are inserted at fixed millisecond timestamps and +-- the rules are evaluated at fixed instants through +-- 'evaluateTimeseriesRules', so every case is fully deterministic. The +-- whole production pipeline below the evaluator loop is exercised: +-- query execution against the store, sample decoding, the per-series +-- @for@ state machine, ingress via the alarm registry, and the history +-- read used for inspection. +module Cardano.Tracer.Test.Alarms.TimeseriesTests + ( tests + ) where + +import Cardano.Logging (SeverityF (..), SeverityS (..)) +import Cardano.Timeseries.API (Config (..), Tree) +import Cardano.Timeseries.Component (TimeseriesConfig (..)) +import qualified Cardano.Timeseries.Component as Timeseries +import Cardano.Tracer.Configuration +import Cardano.Tracer.Handlers.Alarms.Registry +import Cardano.Tracer.Handlers.Alarms.Types +import Cardano.Tracer.MetaTrace (TraceBundle (..), mkTraceBundle) + +import Data.Foldable (for_) +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import Data.Text (Text) +import qualified Data.Text as Text +import Data.Time.Clock (UTCTime) +import Data.Time.Clock.POSIX (posixSecondsToUTCTime) +import Data.Word (Word64) + +import Test.Tasty +import Test.Tasty.QuickCheck hiding (labels) + +tests :: TestTree +tests = testGroup "Test.Alarms.Timeseries" + [ testGroup "mempool-high" + [ testProperty "mempool below threshold raises no alarm" propMempoolQuiet + , testProperty "sustained high mempool raises exactly one alarm" propMempoolAlarm + ] + , testGroup "blockheight-unchanged" + [ testProperty "growing chain raises no alarm" propBlockheightQuiet + , testProperty "stalled blockheight raises exactly one alarm" propBlockheightAlarm + ] + , testGroup "high-ping-latency" + [ testProperty "latency spike shorter than for raises no alarm" propPingQuiet + , testProperty "sustained high latency raises exactly one alarm" propPingAlarm + ] + ] + +-------------------------------------------------------------------------------- +-- Fixture +-------------------------------------------------------------------------------- + +-- | Fixed base timestamp (ms since epoch). Must stay far above the +-- store's 300 s staleness window: the Tree store's window arithmetic +-- is on 'Word64' and underflows for query times below it. +t0 :: Word64 +t0 = 1_700_000_000_000 + +-- | Every sample belongs to one node, labelled the way the acceptor +-- labels forwarded metrics ("Cardano.Tracer.Acceptors.Utils"). +testSeries :: Set.Set (Text, Text) +testSeries = Set.fromList [("node_name", "node-1")] + +-- | Store configuration for tests. The pruner compares against the real +-- wall clock, so the retention window must be large enough that its +-- cutoff (now - retention) stays below the fixed 2023-era sample +-- timestamps — but small enough that the 'Word64' subtraction cannot +-- wrap. 10^12 ms (~32 years) satisfies both for decades. The pruning +-- period must be 'Just': with 'Nothing' the pruner parks forever on an +-- MVar, and once the abandoned test handle becomes garbage the RTS's +-- deadlock detector kills it, rethrowing through 'link' into whatever +-- test runs later. +testTimeseriesConfig :: TimeseriesConfig +testTimeseriesConfig = TimeseriesConfig + { retentionMillis = 1_000_000_000_000 + , pruningPeriodMillis = Just (24 * 60 * 60 * 1000) + , interpCfg = Config { defaultRangeSamplingRateMillis = 15_000 } + } + +mkAlarmsConfig :: AlarmsTimeseriesRuleConfig -> AlarmsConfig +mkAlarmsConfig rule = AlarmsConfig + { alEndpoint = Endpoint "127.0.0.1" 0 Nothing + , alAllowInsecure = Just True + , alRetention = Nothing + , alLimits = Nothing + , alAuthentication = AlarmsAuthConfig [] [] + , alConsumers = [] + , alTraceRules = Nothing + , alTimeseriesRules = Just [rule] + } + +-- | Evaluation timestamp of round @k@: 30 simulated seconds per round +-- after 't0'. +roundTime :: Word64 -> UTCTime +roundTime k = posixSecondsToUTCTime (fromIntegral (t0 `div` 1000 + 30 * k)) + +-- | One sample per 30 s starting at 't0': offsets and values for +-- indices @0 .. count-1@. +samplesEvery30s :: Word64 -> (Word64 -> Double) -> [(Word64, Double)] +samplesEvery30s count valueAt = [ (i * 30_000, valueAt i) | i <- [0 .. count - 1] ] + +-- | Build a store and a registry holding one rule, insert all samples +-- of one metric up front (every query looks strictly backward, so +-- later samples are invisible to earlier rounds), evaluate the rule +-- at rounds @k = 1 .. rounds@, and return the alarm history. +-- +-- Rounds start at @k = 1@: at @k = 0@ a range window holds a single +-- populated grid point and @rate@ has no defined value there. +runScenario + :: AlarmsTimeseriesRuleConfig + -> Text -- ^ metric name + -> [(Word64, Double)] -- ^ samples: (ms after 't0', value) + -> Word64 -- ^ number of 30 s evaluation rounds + -> IO [(AlarmCursor, AlarmEvent)] +runScenario rule metric samples rounds = do + bundle <- mkTraceBundle (SeverityF (Just Warning)) + handle <- Timeseries.create @(Tree Double) (timeseries bundle) (Just testTimeseriesConfig) + registry <- newAlarmRegistry (assorted bundle) (mkAlarmsConfig rule) + for_ samples \(offset, value) -> + Timeseries.insert handle testSeries (t0 + offset) [(metric, value)] + for_ [1 .. rounds] \k -> + evaluateTimeseriesRules registry handle (roundTime k) + readHistoryFiltered registry Nothing 100 emptyAlarmFilter + +-------------------------------------------------------------------------------- +-- Assertions +-------------------------------------------------------------------------------- + +expectNoAlarm :: [(AlarmCursor, AlarmEvent)] -> Property +expectNoAlarm events = + counterexample ("unexpected alarms: " <> show (map snd events)) + (length events === 0) + +-- | Exactly one alarm, carrying the rule identity, the severity, the +-- trusted source, the series label, and a per-series source event id. +expectOneAlarm :: AlarmsTimeseriesRuleConfig -> [(AlarmCursor, AlarmEvent)] -> Property +expectOneAlarm rule events = case events of + [(_, ev)] -> conjoin + [ counterexample "ruleId" (ruleId ev === RuleId (atsRuleId rule)) + , counterexample "severity" (severity ev === atsSeverity rule) + , counterexample "source" (source ev === AlarmSource "timeseries") + , counterexample "summary" (Just (summary ev) === atsSummary rule) + , counterexample "node_name label" + (Map.lookup "node_name" (labels ev) === Just "node-1") + , counterexample ("sourceEventId: " <> show (sourceEventId ev)) + (property (("ts:" <> atsRuleId rule <> ":node_name=node-1:") + `Text.isPrefixOf` sourceEventId ev)) + ] + _ -> counterexample ("expected exactly one alarm, got: " <> show (map snd events)) + (property False) + +-------------------------------------------------------------------------------- +-- mempool-high: instant threshold +-------------------------------------------------------------------------------- + +-- | Catalogue rule @cardano_node_mempool_high@. Note the applied @now@: +-- a bare metric is a function of time in the query language. +mempoolRule :: AlarmsTimeseriesRuleConfig +mempoolRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "mempool-high" + , atsSummary = Just "More than 200 transactions in mempool for over 10 minutes" + , atsSeverity = Warning + , atsQuery = "cardano_node_metrics_txsInMempool_int now > 200" + , atsEvaluateEvery = 30 + , atsFor = Just 600 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +mempoolMetric :: Text +mempoolMetric = "cardano_node_metrics_txsInMempool_int" + +-- | The mempool hovers between 120 and 180 transactions — always below +-- the threshold, so the expression is never true. +propMempoolQuiet :: Property +propMempoolQuiet = once $ ioProperty do + events <- runScenario mempoolRule mempoolMetric + (samplesEvery30s 25 \i -> 120 + fromIntegral (i `mod` 3) * 30) + 24 + pure (expectNoAlarm events) + +-- | The mempool sits at 250 transactions for the whole horizon. The +-- expression turns true at round 1, satisfies @for@ = 600 s at round +-- 21, and the three extra rounds prove the edge stays published only +-- once. +propMempoolAlarm :: Property +propMempoolAlarm = once $ ioProperty do + events <- runScenario mempoolRule mempoolMetric + (samplesEvery30s 25 (const 250)) + 24 + pure (expectOneAlarm mempoolRule events) + +-------------------------------------------------------------------------------- +-- blockheight-unchanged: rate over a range window +-------------------------------------------------------------------------------- + +-- | Catalogue rule @cardano_node_blockheight_unchanged@. The range form +-- needs no applied @now@; @== 0@ keeps exactly the series whose block +-- height did not move inside the 5 min window. +blockheightRule :: AlarmsTimeseriesRuleConfig +blockheightRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "blockheight-unchanged" + , atsSummary = Just "Blockheight unchanged for more than 7 minutes" + , atsSeverity = Critical + , atsQuery = "rate (cardano_node_metrics_blockNum_int[now - 5m; now]) == 0" + , atsEvaluateEvery = 30 + , atsFor = Just 120 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +blockheightMetric :: Text +blockheightMetric = "cardano_node_metrics_blockNum_int" + +-- | The chain grows by one block per sample, so the rate over the +-- window is strictly positive and the comparison filters the series +-- out of the result on every round. +propBlockheightQuiet :: Property +propBlockheightQuiet = once $ ioProperty do + events <- runScenario blockheightRule blockheightMetric + (samplesEvery30s 17 \i -> 1000 + fromIntegral i) + 16 + pure (expectNoAlarm events) + +-- | The block height never moves: the rate is exactly 0 on every round, +-- and after @for@ = 120 s (round 5) exactly one critical alarm is +-- published. +propBlockheightAlarm :: Property +propBlockheightAlarm = once $ ioProperty do + events <- runScenario blockheightRule blockheightMetric + (samplesEvery30s 17 (const 1000)) + 16 + pure (expectOneAlarm blockheightRule events) + +-------------------------------------------------------------------------------- +-- high-ping-latency: avg_over_time over a range window +-------------------------------------------------------------------------------- + +-- | Catalogue rule @high_cardano_ping_latency@ (also the concept doc's +-- own example rule). +pingRule :: AlarmsTimeseriesRuleConfig +pingRule = AlarmsTimeseriesRuleConfig + { atsRuleId = "high-ping-latency" + , atsSummary = Just "Average node ping latency above 500 ms" + , atsSeverity = Warning + , atsQuery = "avg_over_time (netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average[now - 5m; now]) > 500" + , atsEvaluateEvery = 30 + , atsFor = Just 3600 + , atsRepeatEvery = Nothing + , atsLabels = Nothing + } + +pingMetric :: Text +pingMetric = "netdata_statsd_cardano_node_ping_latency_ms_gauge_value_average" + +-- | A 10-minute latency spike (600 ms), then recovery to 100 ms. The +-- expression is true while the spike dominates the trailing 5 min +-- average — far shorter than @for@ = 60 min — then turns false and +-- resets the pending state, so nothing may ever be published. +propPingQuiet :: Property +propPingQuiet = once $ ioProperty do + events <- runScenario pingRule pingMetric + (samplesEvery30s 61 \i -> if i < 20 then 600 else 100) + 60 + pure (expectNoAlarm events) + +-- | Latency stays at 800 ms for 62 minutes. The expression is true from +-- round 1 on, satisfies @for@ = 3600 s at round 121, and exactly one +-- warning alarm is published. +propPingAlarm :: Property +propPingAlarm = once $ ioProperty do + events <- runScenario pingRule pingMetric + (samplesEvery30s 125 (const 800)) + 124 + pure (expectOneAlarm pingRule events) diff --git a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs index 017d1ff43ff..874de8e2f7a 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/DataPoint/Tests.hs @@ -89,6 +89,7 @@ propDataPoint ts@TestSetup{..} rootDir localSock = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams rootDir FileMode ForHuman] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs index 02ea981e441..ff3948a2994 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Logs/Tests.hs @@ -71,6 +71,7 @@ propLogs ts@TestSetup{..} format logRotLimitBytes logRotMaxAgeMinutes rootDir lo , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , logging = LoggingParams root FileMode format :| [] , rotation = Just $ RotationParams { rpFrequencySecs = 3 @@ -119,6 +120,7 @@ propMultiInit ts@TestSetup{..} format rootDir howToConnect1 howToConnect2 = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = LoggingParams rootDir FileMode format :| [] , rotation = Nothing @@ -165,6 +167,7 @@ propMultiResp ts@TestSetup{..} format rootDir howToConnect = do , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = LoggingParams rootDir FileMode format :| [] , rotation = Nothing diff --git a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs index f9969008ca4..0d8d54e9e68 100644 --- a/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs +++ b/cardano-tracer/test/Cardano/Tracer/Test/Restart/Tests.hs @@ -94,6 +94,7 @@ mkConfig TestSetup{..} rootDir p = TracerConfig , hasEKG = Nothing , hasPrometheus = Nothing , hasTimeseries = Nothing + , alarms = Nothing , tlsCertificate = Nothing , logging = NE.fromList [LoggingParams rootDir FileMode ForMachine] , rotation = Nothing diff --git a/cardano-tracer/test/cardano-tracer-test.hs b/cardano-tracer/test/cardano-tracer-test.hs index c862a6770d5..c3db8dcdcba 100644 --- a/cardano-tracer/test/cardano-tracer-test.hs +++ b/cardano-tracer/test/cardano-tracer-test.hs @@ -1,6 +1,8 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# OPTIONS_GHC -Wno-unused-matches #-} +import qualified Cardano.Tracer.Test.Alarms.Tests as Alarms +import qualified Cardano.Tracer.Test.Alarms.TimeseriesTests as AlarmsTimeseries import qualified Cardano.Tracer.Test.DataPoint.Tests as DataPoint import qualified Cardano.Tracer.Test.Logs.Tests as Logs import Cardano.Tracer.Test.TestSetup @@ -47,6 +49,8 @@ main = do (testGroup "Tests" [ Logs.tests ts , DataPoint.tests ts + , Alarms.tests + , AlarmsTimeseries.tests -- , Restart.tests ts -- , Queue.tests ts ])