diff --git a/CHANGELOG.md b/CHANGELOG.md index 31292a1..5eb73f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ workflow refuses a tag whose version has no entry here. ## [Unreleased] +### Added + +- **Alerts to Telegram, Discord or a webhook** (urbforge/urbex-cli#95): `observability.alerts` in `urbex.platform.yaml` provisions a Grafana contact point for Urbex's alerts - one short message per alert and machine, its summary, severity, project, service and host, a link to Grafana, and a resolved message; repeated at most every 4 hours. The bot token and the webhooks' URLs are credentials, encrypted in the observability Stack. `urbex doctor` checks them, and `urbex doctor --test-alerts` sends a test message through each channel. + ## [0.1.0] - 2026-11-17 The first preview release: urbex as binaries for Linux and macOS (amd64, diff --git a/internal/bootstrap/alerts.go b/internal/bootstrap/alerts.go new file mode 100644 index 0000000..317ea3e --- /dev/null +++ b/internal/bootstrap/alerts.go @@ -0,0 +1,122 @@ +package bootstrap + +import ( + "fmt" + "strconv" + "strings" + + "github.com/urbforge/urbex-cli/internal/credentials" + "github.com/urbforge/urbex-cli/internal/platform" +) + +// AlertsProvisioningFile is the Grafana provisioning file, in the +// observability Stack, that sends Urbex's alerts to the channels of +// observability.alerts (urbforge/urbex-cli#95). urbex writes it when a +// channel is configured, and removes it when none is. +const AlertsProvisioningFile = "grafana/provisioning/alerting/urbex-notifications.yml" + +// The secrets the provisioning file reads from Grafana's environment +// (Grafana expands $VAR in provisioning files): the Stack's secrets. +const ( + alertsTelegramTokenVar = "URBEX_ALERTS_TELEGRAM_BOT_TOKEN" + alertsDiscordURLVar = "URBEX_ALERTS_DISCORD_WEBHOOK_URL" + alertsWebhookURLVar = "URBEX_ALERTS_WEBHOOK_URL" +) + +// The grouping and repeat of the alerts' notifications: one message per +// alert and machine, gathered for 30 s, again at most every 4 hours while +// it fires - a flapping disk doesn't fill the chat. +const ( + alertsGroupWait = "30s" + alertsGroupInterval = "5m" + alertsRepeatInterval = "4h" +) + +// alertsMessage is the notifications' text: per alert, firing or +// resolved, its summary and severity, the project, service, env and host +// it is about, and a link to it in Grafana. A rule Grafana couldn't +// evaluate (its data source down) has no values for its summary: the +// message names the rule instead. It is each channel's message itself, +// not a named template: Grafana's Alertmanager loads provisioned +// templates up to a minute after it starts, and a message sent before +// that would be empty. +const alertsMessage = `{{ range .Alerts }}{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }} [{{ or .Labels.severity "info" }}] {{ if or (eq .Labels.alertname "DatasourceError") (eq .Labels.alertname "DatasourceNoData") }}could not evaluate "{{ .Labels.rulename }}" ({{ .Labels.alertname }}){{ else }}{{ or .Annotations.summary .Labels.alertname }}{{ end }}{{ with .Labels.project }} - project {{ . }}{{ end }}{{ with .Labels.service }} - service {{ . }}{{ end }}{{ with .Labels.env }} - {{ . }}{{ end }}{{ with .Labels.host }} - {{ . }}{{ end }} +{{ .GeneratorURL }} +{{ end }}` + +// AlertsProvisioning returns the provisioning file for cfg's alert +// channels and the secrets it reads, or "" when none is configured. +// Missing credentials for a configured channel are an error, naming them. +func AlertsProvisioning(cfg platform.AlertsConfig, creds *credentials.Credentials) (string, map[string]string, error) { + if !cfg.Any() { + return "", nil, nil + } + var missing []string + secrets := map[string]string{} + var receivers strings.Builder + message := strconv.Quote(alertsMessage) + if t := cfg.Telegram; t != nil { + if creds.TelegramBotToken == "" { + missing = append(missing, "telegramBotToken") + } + secrets[alertsTelegramTokenVar] = creds.TelegramBotToken + fmt.Fprintf(&receivers, ` - uid: urbex-telegram + type: telegram + settings: + bottoken: $%s + chatid: %s + message: %s + disable_web_page_preview: true +`, alertsTelegramTokenVar, strconv.Quote(t.ChatID), message) + } + if cfg.Discord { + if creds.DiscordWebhookURL == "" { + missing = append(missing, "discordWebhookUrl") + } + secrets[alertsDiscordURLVar] = creds.DiscordWebhookURL + fmt.Fprintf(&receivers, ` - uid: urbex-discord + type: discord + settings: + url: $%s + message: %s +`, alertsDiscordURLVar, message) + } + if cfg.Webhook { + if creds.AlertsWebhookURL == "" { + missing = append(missing, "alertsWebhookUrl") + } + secrets[alertsWebhookURLVar] = creds.AlertsWebhookURL + fmt.Fprintf(&receivers, ` - uid: urbex-webhook + type: webhook + settings: + url: $%s +`, alertsWebhookURLVar) + } + if len(missing) > 0 { + return "", nil, fmt.Errorf("observability.alerts needs the credential(s) %s: store them with 'urbex-op set ' (see docs/reference/credentials.md in urbforge/urbex)", strings.Join(missing, ", ")) + } + var b strings.Builder + b.WriteString(`# Generated by urbex bootstrap from observability.alerts in +# urbex.platform.yaml (urbforge/urbex-cli#95) - do not edit. Sends the +# alerts of this Grafana to the channels configured there; their secrets +# come from the Stack's encrypted secrets, through Grafana's environment. +apiVersion: 1 +contactPoints: + - orgId: 1 + name: urbex + receivers: +`) + b.WriteString(receivers.String()) + // The whole policy tree: this Grafana is the platform's, and its + // alerts are Urbex's (Grafana 13 has no default contact point to + // route the others to). + fmt.Fprintf(&b, `policies: + - orgId: 1 + receiver: urbex + group_by: [alertname, service, host] + group_wait: %s + group_interval: %s + repeat_interval: %s +`, alertsGroupWait, alertsGroupInterval, alertsRepeatInterval) + return b.String(), secrets, nil +} diff --git a/internal/bootstrap/alerts_test.go b/internal/bootstrap/alerts_test.go new file mode 100644 index 0000000..028f7f4 --- /dev/null +++ b/internal/bootstrap/alerts_test.go @@ -0,0 +1,78 @@ +package bootstrap + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/urbforge/urbex-cli/internal/credentials" + "github.com/urbforge/urbex-cli/internal/platform" +) + +func TestAlertsProvisioning_NoneConfigured(t *testing.T) { + file, secrets, err := AlertsProvisioning(platform.AlertsConfig{}, &credentials.Credentials{}) + if file != "" || secrets != nil || err != nil { + t.Fatalf("got %q %v %v, want nothing", file, secrets, err) + } +} + +func TestAlertsProvisioning_MissingCredentials(t *testing.T) { + cfg := platform.AlertsConfig{Telegram: &platform.TelegramAlerts{ChatID: "-100123"}, Webhook: true} + _, _, err := AlertsProvisioning(cfg, &credentials.Credentials{}) + if err == nil || !strings.Contains(err.Error(), "telegramBotToken, alertsWebhookUrl") { + t.Fatalf("err = %v, want both credentials named", err) + } +} + +func TestAlertsProvisioning_AllChannels(t *testing.T) { + cfg := platform.AlertsConfig{Telegram: &platform.TelegramAlerts{ChatID: "-100123"}, Discord: true, Webhook: true} + creds := &credentials.Credentials{TelegramBotToken: "123:secret-token", DiscordWebhookURL: "https://discord.example.com/api/webhooks/1/secret", AlertsWebhookURL: "https://hooks.example.com/secret"} + file, secrets, err := AlertsProvisioning(cfg, creds) + if err != nil { + t.Fatal(err) + } + for _, v := range []string{creds.TelegramBotToken, creds.DiscordWebhookURL, creds.AlertsWebhookURL} { + if strings.Contains(file, v) { + t.Errorf("secret %q in clear in the provisioning file", v) + } + } + if secrets[alertsTelegramTokenVar] != creds.TelegramBotToken || secrets[alertsDiscordURLVar] != creds.DiscordWebhookURL || secrets[alertsWebhookURLVar] != creds.AlertsWebhookURL { + t.Errorf("secrets = %v", secrets) + } + var doc struct { + APIVersion int `yaml:"apiVersion"` + ContactPoints []struct { + Name string + Receivers []struct { + UID, Type string + Settings map[string]any + } + } `yaml:"contactPoints"` + Policies []struct { + Receiver string + GroupBy []string `yaml:"group_by"` + RepeatInterval string `yaml:"repeat_interval"` + } + } + if err := yaml.Unmarshal([]byte(file), &doc); err != nil { + t.Fatalf("not YAML: %v\n%s", err, file) + } + if doc.APIVersion != 1 || strings.Contains(file, "templates:") { + t.Errorf("apiVersion %d, or a named template (loaded late by Grafana)", doc.APIVersion) + } + if len(doc.ContactPoints) != 1 || len(doc.ContactPoints[0].Receivers) != 3 { + t.Fatalf("contact points: %+v", doc.ContactPoints) + } + tg := doc.ContactPoints[0].Receivers[0] + if tg.Type != "telegram" || tg.Settings["bottoken"] != "$"+alertsTelegramTokenVar || tg.Settings["chatid"] != "-100123" || tg.Settings["message"] != alertsMessage { + t.Errorf("telegram receiver: %+v", tg) + } + if doc.ContactPoints[0].Receivers[1].Settings["url"] != "$"+alertsDiscordURLVar { + t.Errorf("discord receiver: %+v", doc.ContactPoints[0].Receivers[1]) + } + r := doc.Policies[0] + if r.Receiver != "urbex" || r.RepeatInterval != "4h" || strings.Join(r.GroupBy, ",") != "alertname,service,host" { + t.Errorf("route: %+v", r) + } +} diff --git a/internal/bootstrap/orchestrator.go b/internal/bootstrap/orchestrator.go index 353183e..c37c468 100644 --- a/internal/bootstrap/orchestrator.go +++ b/internal/bootstrap/orchestrator.go @@ -197,7 +197,10 @@ func (o *Orchestrator) Run(ctx context.Context, opts Options) error { // The platform services Komodo deploys from the GitOps repo: their // folders go in before setupPlatform pushes it. o.Progress.Begin(PhaseGitOps) - stacks := o.platformStacks(tunnelToken) + stacks, err := o.platformStacks(tunnelToken) + if err != nil { + return err + } if err := o.writePlatformStacks(ctx, opts.GitOpsRepoPath, stacks); err != nil { return err } diff --git a/internal/bootstrap/platform.go b/internal/bootstrap/platform.go index 1a82b01..226ed50 100644 --- a/internal/bootstrap/platform.go +++ b/internal/bootstrap/platform.go @@ -97,6 +97,8 @@ type Endpoints struct { TechnitiumURL string // TechnitiumIP is Technitium's address: every LXC's first nameserver. TechnitiumIP string + // Grafana is the observability stack's Grafana (http://host:3000). + Grafana string } // InternalEndpointsFor computes the base services' endpoints by internal @@ -116,6 +118,7 @@ func InternalEndpointsFor(cfg *platform.Config) (Endpoints, error) { Registry: name("git") + ":3000", KomodoURL: "http://" + name("komodo") + ":9120", LokiPushURL: "http://" + name("loki") + ":3100/loki/api/v1/push", + Grafana: "http://" + name("grafana") + ":3000", PrometheusWriteURL: "http://" + name("prometheus") + ":9090/api/v1/write", KeycloakURL: "http://" + name("auth") + ":8080", TechnitiumURL: "http://" + name("dns") + ":5380", @@ -135,6 +138,7 @@ func EndpointsFor(cfg *platform.Config) (Endpoints, error) { Registry: giteaIP + ":3000", KomodoURL: "http://" + ips[Hostname("komodo")] + ":9120", LokiPushURL: "http://" + ips[Hostname("observability")] + ":3100/loki/api/v1/push", + Grafana: "http://" + ips[Hostname("observability")] + ":3000", PrometheusWriteURL: "http://" + ips[Hostname("observability")] + ":9090/api/v1/write", KeycloakURL: "http://" + ips[Hostname("keycloak")] + ":8080", TechnitiumURL: "http://" + ips[Hostname("technitium")] + ":5380", diff --git a/internal/bootstrap/stacks.go b/internal/bootstrap/stacks.go index c206f57..d68d017 100644 --- a/internal/bootstrap/stacks.go +++ b/internal/bootstrap/stacks.go @@ -14,6 +14,7 @@ import ( "strings" "sync" + "github.com/urbforge/urbex-cli/internal/platform" assets "github.com/urbforge/urbex-cli/platform" "github.com/urbforge/urbex-cli/internal/komodo" @@ -34,6 +35,10 @@ type PlatformStack struct { Name, Dir string Config map[string]string Secrets map[string]string + // Generated are files urbex writes besides the embedded ones, by + // their path in the Stack's folder; a path mapped to "" is removed, + // so that a file no longer wanted doesn't stay behind. + Generated map[string]string } // ComposeWrapper is the command every Stack deployed from the GitOps repo @@ -52,7 +57,7 @@ const PlatformStacksDir = "platform" // platformStacks are the platform services deployed from the GitOps // repo: Keycloak, the observability stack and, with Cloudflare, the // tunnel, whose connector token is tunnelToken. -func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack { +func (o *Orchestrator) platformStacks(tunnelToken string) ([]PlatformStack, error) { // Keycloak runs in production mode on its own PostgreSQL, in the same // Stack (urbex-cli#36). Both containers read config.env and the // secrets: KC_* configure Keycloak, POSTGRES_* the database. @@ -89,14 +94,45 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack { "KC_DB_PASSWORD": o.Credentials.KeycloakDBPassword, "POSTGRES_PASSWORD": o.Credentials.KeycloakDBPassword, }}, - {Name: Hostname("observability"), Dir: "observability", Config: map[string]string{}, - Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}}, } + observability, err := o.observabilityStack() + if err != nil { + return nil, err + } + stacks = append(stacks, observability) if o.Config.Cloudflare.Enabled() { stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{}, Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}}) } - return stacks + return stacks, nil +} + +// observabilityStack is Prometheus, Loki and Grafana; with +// observability.alerts, Grafana also sends Urbex's alerts to its channels +// (urbforge/urbex-cli#95), and its links point to its own address. +func (o *Orchestrator) observabilityStack() (PlatformStack, error) { + st := PlatformStack{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{}, + Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}, + Generated: map[string]string{AlertsProvisioningFile: ""}} + file, secrets, err := AlertsProvisioning(o.Config.Observability.Alerts, o.Credentials) + if err != nil || file == "" { + return st, err + } + st.Generated[AlertsProvisioningFile] = file + for k, v := range secrets { + st.Secrets[k] = v + } + st.Config["GF_SERVER_ROOT_URL"] = GrafanaURL(o.Config) + return st, nil +} + +// GrafanaURL is the address the alerts link to: Grafana behind Access with +// Cloudflare, else its internal name, on the LAN or the VPN. +func GrafanaURL(cfg *platform.Config) string { + if host := PublicEndpointsFor(cfg).Grafana; host != "" { + return "https://" + host + "/" + } + return "http://" + cfg.InternalName("grafana") + ":3000/" } // Keycloak's database and its owner, in the Keycloak Stack's PostgreSQL. @@ -144,6 +180,29 @@ func (o *Orchestrator) writePlatformStacks(ctx context.Context, gitopsRepoPath s if err != nil { return fmt.Errorf("writing the %s stack: %w", st.Dir, err) } + generated := make([]string, 0, len(st.Generated)) + for p := range st.Generated { + generated = append(generated, p) + } + sort.Strings(generated) + for _, p := range generated { + dest := filepath.Join(dir, filepath.FromSlash(p)) + data := st.Generated[p] + if data == "" { + if err := os.Remove(dest); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + continue + } + fmt.Fprintf(files, "%s %d\n", p, len(data)) + files.Write([]byte(data)) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + if err := os.WriteFile(dest, []byte(data), 0o644); err != nil { + return err + } + } // The files' hash in config.env, which the containers that read // the files also read: compose recreates them when a file // changes. A container keeps a single bind-mounted file as it diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 2582353..51a16ff 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -8,9 +8,11 @@ import ( "github.com/spf13/cobra" + "github.com/urbforge/urbex-cli/internal/bootstrap" "github.com/urbforge/urbex-cli/internal/cloudflare" "github.com/urbforge/urbex-cli/internal/credentials" "github.com/urbforge/urbex-cli/internal/doctor" + "github.com/urbforge/urbex-cli/internal/grafana" "github.com/urbforge/urbex-cli/internal/platform" "github.com/urbforge/urbex-cli/internal/proxmoxapi" "github.com/urbforge/urbex-cli/internal/version" @@ -22,6 +24,7 @@ var errDoctorFailed = fmt.Errorf("some checks failed") func newDoctorCmd() *cobra.Command { var gitopsRepoPath string + var testAlerts bool var output outputFlag cmd := &cobra.Command{ @@ -32,11 +35,13 @@ ansible-playbook, sops, age); the workspace (the credentials, the SSH key, the GitOps repo and its urbex.platform.yaml); with a platform config and a Proxmox token, the node (the API answers, the node, pool, storage, LXC template and bridge exist, the token has the privileges urbex needs); and, -with Cloudflare in the config, the token and the zone. +with Cloudflare in the config, the token and the zone; with +observability.alerts, the channels' credentials. It starts with urbex's own version. Each check is ok, warn, fail or skip, with what to do when it isn't ok. It exits non-zero when a check failed. It changes nothing, and prints no -secret. --output json prints the report as one JSON document. See +secret. --test-alerts also sends a test message through each alert +channel, from Grafana. --output json prints the report as one JSON document. See docs/reference/cli.md in urbforge/urbex.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { @@ -78,6 +83,15 @@ docs/reference/cli.md in urbforge/urbex.`, cf = cloudflare.New(cloudflare.DefaultBaseURL, creds.CloudflareToken, cfg.Cloudflare.AccountID, cfg.Cloudflare.ZoneID) } doctor.CloudflareChecks(ctx, cf, cfg, r) + var at doctor.AlertsTester + if cfg != nil && testAlerts { + ep, err := bootstrap.EndpointsFor(cfg) + if err != nil { + return err + } + at = grafana.New(ep.Grafana, "admin", creds.GrafanaAdminPassword) + } + doctor.AlertsChecks(ctx, at, cfg, creds, r) r.OK = !r.Failed() out := cmd.OutOrStdout() @@ -94,6 +108,7 @@ docs/reference/cli.md in urbforge/urbex.`, return nil }, } + cmd.Flags().BoolVar(&testAlerts, "test-alerts", false, "send a test message through each channel of observability.alerts") cmd.Flags().StringVar(&gitopsRepoPath, "gitops-repo", "", "local working directory of the GitOps repo (or set URBEX_GITOPS_REPO)") output.Register(cmd) return cmd diff --git a/internal/credentials/credentials.go b/internal/credentials/credentials.go index c139254..9b4c1eb 100644 --- a/internal/credentials/credentials.go +++ b/internal/credentials/credentials.go @@ -27,6 +27,11 @@ type Credentials struct { ProxmoxToken string `yaml:"proxmoxToken,omitempty"` CloudflareToken string `yaml:"cloudflareToken,omitempty"` AgeKey string `yaml:"ageKey,omitempty"` + // Where Grafana sends the alerts (observability.alerts in the + // platform config, urbforge/urbex-cli#95): provided by the user. + TelegramBotToken string `yaml:"telegramBotToken,omitempty"` + DiscordWebhookURL string `yaml:"discordWebhookUrl,omitempty"` + AlertsWebhookURL string `yaml:"alertsWebhookUrl,omitempty"` // Generated by urbex bootstrap. ForgeAdminPassword string `yaml:"forgeAdminPassword,omitempty"` @@ -58,6 +63,9 @@ func (c *Credentials) fields() []struct { {"URBEX_PROXMOX_TOKEN", &c.ProxmoxToken}, {"URBEX_CLOUDFLARE_TOKEN", &c.CloudflareToken}, {"URBEX_AGE_KEY", &c.AgeKey}, + {"URBEX_TELEGRAM_BOT_TOKEN", &c.TelegramBotToken}, + {"URBEX_DISCORD_WEBHOOK_URL", &c.DiscordWebhookURL}, + {"URBEX_ALERTS_WEBHOOK_URL", &c.AlertsWebhookURL}, {"URBEX_FORGE_ADMIN_PASSWORD", &c.ForgeAdminPassword}, {"URBEX_FORGE_TOKEN", &c.ForgeToken}, {"URBEX_KOMODO_ADMIN_PASSWORD", &c.KomodoAdminPassword}, @@ -309,6 +317,10 @@ var provided = map[string]bool{ "URBEX_PROXMOX_TOKEN": true, "URBEX_CLOUDFLARE_TOKEN": true, "URBEX_AGE_KEY": true, + // The alerts' channels. + "URBEX_TELEGRAM_BOT_TOKEN": true, + "URBEX_DISCORD_WEBHOOK_URL": true, + "URBEX_ALERTS_WEBHOOK_URL": true, } // ClearGenerated forgets every credential urbex bootstrap generated - diff --git a/internal/doctor/alerts.go b/internal/doctor/alerts.go new file mode 100644 index 0000000..dd9a794 --- /dev/null +++ b/internal/doctor/alerts.go @@ -0,0 +1,85 @@ +package doctor + +import ( + "context" + "strings" + + "github.com/urbforge/urbex-cli/internal/credentials" + "github.com/urbforge/urbex-cli/internal/grafana" + "github.com/urbforge/urbex-cli/internal/platform" +) + +// AlertsTester sends a test notification through a Grafana contact point +// (grafana.Client). +type AlertsTester interface { + TestContactPoint(ctx context.Context, title, summary string) ([]grafana.TestResult, error) +} + +// AlertsChecks checks the channels of observability.alerts: their +// credentials and, with t (urbex doctor --test-alerts), a test message +// through each of them. Without channels it reports that the alerts stay +// in Grafana. +func AlertsChecks(ctx context.Context, t AlertsTester, cfg *platform.Config, creds *credentials.Credentials, r *Report) { + add := func(name, status, detail, fix string) { + r.add(Check{Group: "alerts", Name: name, Status: status, Detail: detail, Fix: fix}) + } + if cfg == nil { + return + } + a := cfg.Observability.Alerts + if !a.Any() { + add("channels", Skip, "none in observability.alerts: the alerts stay in Grafana's Alerting page", "") + return + } + creds2 := []struct { + on bool + name, key string + value string + }{ + {a.Telegram != nil, "telegram", "telegramBotToken", creds.TelegramBotToken}, + {a.Discord, "discord", "discordWebhookUrl", creds.DiscordWebhookURL}, + {a.Webhook, "webhook", "alertsWebhookUrl", creds.AlertsWebhookURL}, + } + missing := false + for _, c := range creds2 { + if !c.on { + continue + } + if c.value == "" { + missing = true + add(c.name, Fail, "no "+c.key+" in the credentials", + "store it with 'urbex-op set "+c.key+"' (docs/reference/credentials.md), then 'urbex bootstrap'") + continue + } + if t == nil { + add(c.name, OK, c.key+" set; 'urbex doctor --test-alerts' sends a test message", "") + } + } + if t == nil || missing { + return + } + results, err := t.TestContactPoint(ctx, "urbex", "A test from urbex doctor --test-alerts: this channel gets Urbex's alerts.") + if err != nil { + add("test", Fail, firstLine(err.Error()), "run 'urbex bootstrap' after adding observability.alerts, and check that Grafana is up ('urbex status')") + return + } + for _, res := range results { + if res.OK { + add(res.Type, OK, "test message sent", "") + continue + } + add(res.Type, Fail, "test message failed: "+firstLine(res.Error), alertsFix(res.Type, res.Error)) + } +} + +func alertsFix(kind, err string) string { + switch { + case kind == "telegram" && strings.Contains(err, "401"): + return "the bot token is wrong: copy it again from BotFather, 'urbex-op set telegramBotToken', then 'urbex bootstrap'" + case kind == "telegram" && (strings.Contains(err, "chat not found") || strings.Contains(err, "400")): + return "check observability.alerts.telegram.chatId, and that the bot is a member of that chat" + case kind == "discord": + return "check the channel's webhook (Server Settings > Integrations > Webhooks), 'urbex-op set discordWebhookUrl', then 'urbex bootstrap'" + } + return "check the channel's address, and that Grafana's LXC reaches it" +} diff --git a/internal/doctor/alerts_test.go b/internal/doctor/alerts_test.go new file mode 100644 index 0000000..fef8cde --- /dev/null +++ b/internal/doctor/alerts_test.go @@ -0,0 +1,73 @@ +package doctor + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/urbforge/urbex-cli/internal/credentials" + "github.com/urbforge/urbex-cli/internal/grafana" + "github.com/urbforge/urbex-cli/internal/platform" +) + +type fakeTester struct { + res []grafana.TestResult + err error + n int +} + +func (f *fakeTester) TestContactPoint(context.Context, string, string) ([]grafana.TestResult, error) { + f.n++ + return f.res, f.err +} + +func alertsOf(r *Report) string { + var b []string + for _, c := range r.Checks { + b = append(b, c.Group+"/"+c.Name+"="+c.Status) + } + return strings.Join(b, " ") +} + +func TestAlertsChecks(t *testing.T) { + ctx := context.Background() + tg := &platform.Config{Observability: platform.ObservabilityConfig{Alerts: platform.AlertsConfig{Telegram: &platform.TelegramAlerts{ChatID: "-1"}, Discord: true}}} + full := &credentials.Credentials{TelegramBotToken: "t", DiscordWebhookURL: "d"} + + r := &Report{} + AlertsChecks(ctx, nil, &platform.Config{}, full, r) + if alertsOf(r) != "alerts/channels=skip" { + t.Errorf("none configured: %s", alertsOf(r)) + } + + r = &Report{} + AlertsChecks(ctx, nil, tg, &credentials.Credentials{TelegramBotToken: "t"}, r) + if alertsOf(r) != "alerts/telegram=ok alerts/discord=fail" || !strings.Contains(r.Checks[1].Fix, "urbex-op set discordWebhookUrl") { + t.Errorf("missing credential: %s %+v", alertsOf(r), r.Checks) + } + + // A missing credential: no test is sent. + f := &fakeTester{} + r = &Report{} + AlertsChecks(ctx, f, tg, &credentials.Credentials{TelegramBotToken: "t"}, r) + if f.n != 0 { + t.Error("tested with a credential missing") + } + + f = &fakeTester{res: []grafana.TestResult{ + {Integration: grafana.Integration{Type: "telegram"}, Error: "webhook response status 401 Unauthorized"}, + {Integration: grafana.Integration{Type: "discord"}, OK: true}, + }} + r = &Report{} + AlertsChecks(ctx, f, tg, full, r) + if alertsOf(r) != "alerts/telegram=fail alerts/discord=ok" || !strings.Contains(r.Checks[0].Fix, "BotFather") { + t.Errorf("test results: %s %+v", alertsOf(r), r.Checks) + } + + r = &Report{} + AlertsChecks(ctx, &fakeTester{err: errors.New("grafana has no contact point urbex")}, tg, full, r) + if alertsOf(r) != "alerts/test=fail" || !strings.Contains(r.Checks[0].Fix, "urbex bootstrap") { + t.Errorf("no contact point: %s", alertsOf(r)) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index f8b4d96..4f2c07d 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -25,7 +25,7 @@ const ( // Check is one line of the report. type Check struct { - Group string `json:"group"` // tools, workspace, proxmox, cloudflare + Group string `json:"group"` // tools, workspace, proxmox, cloudflare, alerts Name string `json:"name"` Status string `json:"status"` Detail string `json:"detail,omitempty"` diff --git a/internal/grafana/grafana.go b/internal/grafana/grafana.go new file mode 100644 index 0000000..d79dae4 --- /dev/null +++ b/internal/grafana/grafana.go @@ -0,0 +1,125 @@ +// Package grafana talks to the platform's Grafana: what urbex doctor +// needs of its alerting (urbforge/urbex-cli#95). +package grafana + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Client is Grafana's HTTP API, as its admin. +type Client struct { + BaseURL, User, Password string + HTTP *http.Client +} + +// New returns a client for Grafana at baseURL (http://host:3000). +func New(baseURL, user, password string) *Client { + return &Client{BaseURL: strings.TrimRight(baseURL, "/"), User: user, Password: password, + HTTP: &http.Client{Timeout: 30 * time.Second}} +} + +// Integration is one channel of a contact point. +type Integration struct { + UID string `json:"uid"` + Type string `json:"type"` + raw json.RawMessage +} + +// TestResult is what a channel answered a test message: Error is +// Grafana's, with the channel's URL redacted. +type TestResult struct { + Integration + OK bool + Error string +} + +const receiversPath = "/apis/notifications.alerting.grafana.app/v1beta1/namespaces/default/receivers" + +// receiver is one contact point, in Grafana 13's receivers API. +type receiver struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Title string `json:"title"` + Integrations []json.RawMessage `json:"integrations"` + } `json:"spec"` +} + +// TestContactPoint sends a test notification through each channel of the +// contact point named title, with the settings and secrets Grafana holds. +// It returns an error when Grafana can't be reached or has no such +// contact point. +func (c *Client) TestContactPoint(ctx context.Context, title, summary string) ([]TestResult, error) { + var list struct { + Items []receiver `json:"items"` + } + if err := c.do(ctx, http.MethodGet, receiversPath, nil, &list); err != nil { + return nil, err + } + var rc *receiver + for i := range list.Items { + if list.Items[i].Spec.Title == title { + rc = &list.Items[i] + } + } + if rc == nil { + return nil, fmt.Errorf("grafana has no contact point %s", title) + } + var results []TestResult + for _, raw := range rc.Spec.Integrations { + in := Integration{raw: raw} + if err := json.Unmarshal(raw, &in); err != nil { + return nil, err + } + body := map[string]any{ + "integration": in.raw, + "alert": map[string]any{ + "labels": map[string]string{"alertname": "urbex doctor test", "severity": "info"}, + "annotations": map[string]string{"summary": summary}, + }, + } + var res struct { + Status, Error string + } + if err := c.do(ctx, http.MethodPost, receiversPath+"/"+rc.Metadata.Name+"/test", body, &res); err != nil { + return nil, err + } + results = append(results, TestResult{Integration: in, OK: res.Status == "success", Error: res.Error}) + } + return results, nil +} + +func (c *Client) do(ctx context.Context, method, path string, in, out any) error { + var body io.Reader + if in != nil { + b, err := json.Marshal(in) + if err != nil { + return err + } + body = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, body) + if err != nil { + return err + } + req.SetBasicAuth(c.User, c.Password) + req.Header.Set("Content-Type", "application/json") + resp, err := c.HTTP.Do(req) + if err != nil { + return fmt.Errorf("grafana: %w", err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode/100 != 2 { + return fmt.Errorf("grafana: %s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(data))) + } + return json.Unmarshal(data, out) +} diff --git a/internal/grafana/grafana_test.go b/internal/grafana/grafana_test.go new file mode 100644 index 0000000..96e2081 --- /dev/null +++ b/internal/grafana/grafana_test.go @@ -0,0 +1,65 @@ +package grafana + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestTestContactPoint(t *testing.T) { + var tested []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if u, p, _ := r.BasicAuth(); u != "admin" || p != "pw" { + w.WriteHeader(http.StatusUnauthorized) + return + } + switch { + case r.Method == http.MethodGet && r.URL.Path == receiversPath: + w.Write([]byte(`{"items":[{"metadata":{"name":"ZW1wdHk"},"spec":{"title":"empty","integrations":[]}}, + {"metadata":{"name":"dXJiZXg"},"spec":{"title":"urbex","integrations":[ + {"uid":"urbex-telegram","type":"telegram","settings":{"chatid":"-1"},"secureFields":{"bottoken":true}}, + {"uid":"urbex-webhook","type":"webhook","settings":{},"secureFields":{"url":true}}]}}]}`)) + case r.Method == http.MethodPost && r.URL.Path == receiversPath+"/dXJiZXg/test": + var body struct { + Integration struct { + UID string `json:"uid"` + SecureFields map[string]bool `json:"secureFields"` + } + Alert struct{ Annotations map[string]string } + } + json.NewDecoder(r.Body).Decode(&body) + if len(body.Integration.SecureFields) == 0 || body.Alert.Annotations["summary"] != "hello" { + t.Errorf("test body: %+v", body) + } + tested = append(tested, body.Integration.UID) + if body.Integration.UID == "urbex-telegram" { + w.Write([]byte(`{"status":"failure","error":"failed to send telegram message: webhook response status 401 Unauthorized"}`)) + return + } + w.Write([]byte(`{"status":"success"}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := New(srv.URL, "admin", "pw") + res, err := c.TestContactPoint(context.Background(), "urbex", "hello") + if err != nil { + t.Fatal(err) + } + if len(res) != 2 || res[0].Type != "telegram" || res[0].OK || !strings.Contains(res[0].Error, "401") || !res[1].OK { + t.Errorf("results: %+v", res) + } + if strings.Join(tested, ",") != "urbex-telegram,urbex-webhook" { + t.Errorf("tested %v", tested) + } + if _, err := c.TestContactPoint(context.Background(), "nope", "hello"); err == nil || !strings.Contains(err.Error(), "no contact point nope") { + t.Errorf("missing contact point: %v", err) + } + if _, err := New(srv.URL, "admin", "bad").TestContactPoint(context.Background(), "urbex", "x"); err == nil || !strings.Contains(err.Error(), "401") { + t.Errorf("bad password: %v", err) + } +} diff --git a/internal/platform/config.go b/internal/platform/config.go index f14f8b2..59599d1 100644 --- a/internal/platform/config.go +++ b/internal/platform/config.go @@ -9,6 +9,7 @@ import ( "maps" "net" "os" + "regexp" "slices" "strings" @@ -24,8 +25,38 @@ type Config struct { DNS DNSConfig `yaml:"dns"` Keycloak KeycloakConfig `yaml:"keycloak"` Storage StorageConfig `yaml:"storage,omitempty"` + // Observability is about the observability stack (Grafana). + Observability ObservabilityConfig `yaml:"observability,omitempty"` } +// ObservabilityConfig is about the observability stack. +type ObservabilityConfig struct { + Alerts AlertsConfig `yaml:"alerts,omitempty"` +} + +// AlertsConfig is where Grafana sends Urbex's alerts besides its own +// Alerting page (urbforge/urbex-cli#95): none, one or several channels. +// Their secrets - the bot token, the webhooks' URLs - are credentials +// (credentials.yaml), never here. +type AlertsConfig struct { + // Telegram sends them to a chat through a bot: ChatID here, the bot's + // token as the credential telegramBotToken. + Telegram *TelegramAlerts `yaml:"telegram,omitempty"` + // Discord sends them to a channel's webhook, the credential + // discordWebhookUrl. + Discord bool `yaml:"discord,omitempty"` + // Webhook posts them as JSON to the credential alertsWebhookUrl. + Webhook bool `yaml:"webhook,omitempty"` +} + +// TelegramAlerts is the chat a Telegram bot sends the alerts to. +type TelegramAlerts struct { + ChatID string `yaml:"chatId"` +} + +// Any reports whether a channel is configured. +func (a AlertsConfig) Any() bool { return a.Telegram != nil || a.Discord || a.Webhook } + // StorageConfig is storage that exists outside Urbex (ADR-0028). type StorageConfig struct { // Shares are folders the operator provides - a NAS mounted on the @@ -337,6 +368,13 @@ func (c *Config) Validate() []string { errs = append(errs, "storage.shares."+name+".path: "+p+" is not a folder on the Proxmox node") } } + if t := c.Observability.Alerts.Telegram; t != nil && !telegramChat.MatchString(t.ChatID) { + errs = append(errs, "observability.alerts.telegram.chatId: "+t.ChatID+" is not a Telegram chat id (a number, negative for a group) or @channel") + } return errs } + +// telegramChat is a Telegram chat id: a number (negative for a group) or +// a public channel's @name. +var telegramChat = regexp.MustCompile(`^(-?[0-9]+|@[A-Za-z][A-Za-z0-9_]{4,})$`) diff --git a/internal/platform/config_test.go b/internal/platform/config_test.go index ebdcdbd..c62ea5d 100644 --- a/internal/platform/config_test.go +++ b/internal/platform/config_test.go @@ -269,3 +269,17 @@ func TestLoadFile_LegacyGiteaURL(t *testing.T) { } } } + +func TestValidate_AlertsTelegramChatID(t *testing.T) { + for chat, ok := range map[string]bool{"-1001234567890": true, "123456": true, "@urbex_alerts": true, "my chat": false, "": false, "@ab": false} { + c := Config{Observability: ObservabilityConfig{Alerts: AlertsConfig{Telegram: &TelegramAlerts{ChatID: chat}}}} + errs := c.Validate() + got := !strings.Contains(strings.Join(errs, "\n"), "observability.alerts.telegram.chatId") + if got != ok { + t.Errorf("chatId %q: valid=%v, want %v (%v)", chat, got, ok, errs) + } + } + if (AlertsConfig{}).Any() || !(AlertsConfig{Discord: true}).Any() { + t.Error("AlertsConfig.Any is wrong") + } +} diff --git a/platform/stacks/observability/grafana/provisioning/alerting/urbex-reconcile.yml b/platform/stacks/observability/grafana/provisioning/alerting/urbex-reconcile.yml index 55f78bd..97a9ea2 100644 --- a/platform/stacks/observability/grafana/provisioning/alerting/urbex-reconcile.yml +++ b/platform/stacks/observability/grafana/provisioning/alerting/urbex-reconcile.yml @@ -40,7 +40,7 @@ groups: noDataState: Alerting execErrState: Error for: 0s - labels: { kind: platform, service: komodo } + labels: { kind: platform, service: komodo, severity: critical } annotations: summary: The Komodo Action urbex-reconcile has not run for five minutes. description: Is it scheduled, and is Komodo Core up? Until it runs, an application that uses another and goes down stays down. @@ -70,7 +70,7 @@ groups: noDataState: OK execErrState: Error for: 0s - labels: { kind: platform, service: komodo } + labels: { kind: platform, service: komodo, severity: warning } annotations: summary: The reconciler holds applications back. description: 'Its error lines in Grafana ({source="urbex-reconcile", level="error"}) and urbex status say which: a circuit open, a provider restarting.'