Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ workflow refuses a tag whose version has no entry here.

## [Unreleased]

### Added

- **Alerts to Telegram, Discord or a webhook** (urbforge/urbex-cli#95): `observability.alerts` in `urbex.platform.yaml` provisions a Grafana contact point for Urbex's alerts - one short message per alert and machine, its summary, severity, project, service and host, a link to Grafana, and a resolved message; repeated at most every 4 hours. The bot token and the webhooks' URLs are credentials, encrypted in the observability Stack. `urbex doctor` checks them, and `urbex doctor --test-alerts` sends a test message through each channel.

## [0.1.0] - 2026-11-17

The first preview release: urbex as binaries for Linux and macOS (amd64,
Expand Down
122 changes: 122 additions & 0 deletions internal/bootstrap/alerts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package bootstrap

import (
"fmt"
"strconv"
"strings"

"github.com/urbforge/urbex-cli/internal/credentials"
"github.com/urbforge/urbex-cli/internal/platform"
)

// AlertsProvisioningFile is the Grafana provisioning file, in the
// observability Stack, that sends Urbex's alerts to the channels of
// observability.alerts (urbforge/urbex-cli#95). urbex writes it when a
// channel is configured, and removes it when none is.
const AlertsProvisioningFile = "grafana/provisioning/alerting/urbex-notifications.yml"

// The secrets the provisioning file reads from Grafana's environment
// (Grafana expands $VAR in provisioning files): the Stack's secrets.
const (
alertsTelegramTokenVar = "URBEX_ALERTS_TELEGRAM_BOT_TOKEN"
alertsDiscordURLVar = "URBEX_ALERTS_DISCORD_WEBHOOK_URL"
alertsWebhookURLVar = "URBEX_ALERTS_WEBHOOK_URL"
)

// The grouping and repeat of the alerts' notifications: one message per
// alert and machine, gathered for 30 s, again at most every 4 hours while
// it fires - a flapping disk doesn't fill the chat.
const (
alertsGroupWait = "30s"
alertsGroupInterval = "5m"
alertsRepeatInterval = "4h"
)

// alertsMessage is the notifications' text: per alert, firing or
// resolved, its summary and severity, the project, service, env and host
// it is about, and a link to it in Grafana. A rule Grafana couldn't
// evaluate (its data source down) has no values for its summary: the
// message names the rule instead. It is each channel's message itself,
// not a named template: Grafana's Alertmanager loads provisioned
// templates up to a minute after it starts, and a message sent before
// that would be empty.
const alertsMessage = `{{ range .Alerts }}{{ if eq .Status "firing" }}FIRING{{ else }}RESOLVED{{ end }} [{{ or .Labels.severity "info" }}] {{ if or (eq .Labels.alertname "DatasourceError") (eq .Labels.alertname "DatasourceNoData") }}could not evaluate "{{ .Labels.rulename }}" ({{ .Labels.alertname }}){{ else }}{{ or .Annotations.summary .Labels.alertname }}{{ end }}{{ with .Labels.project }} - project {{ . }}{{ end }}{{ with .Labels.service }} - service {{ . }}{{ end }}{{ with .Labels.env }} - {{ . }}{{ end }}{{ with .Labels.host }} - {{ . }}{{ end }}
{{ .GeneratorURL }}
{{ end }}`

// AlertsProvisioning returns the provisioning file for cfg's alert
// channels and the secrets it reads, or "" when none is configured.
// Missing credentials for a configured channel are an error, naming them.
func AlertsProvisioning(cfg platform.AlertsConfig, creds *credentials.Credentials) (string, map[string]string, error) {
if !cfg.Any() {
return "", nil, nil
}
var missing []string
secrets := map[string]string{}
var receivers strings.Builder
message := strconv.Quote(alertsMessage)
if t := cfg.Telegram; t != nil {
if creds.TelegramBotToken == "" {
missing = append(missing, "telegramBotToken")
}
secrets[alertsTelegramTokenVar] = creds.TelegramBotToken
fmt.Fprintf(&receivers, ` - uid: urbex-telegram
type: telegram
settings:
bottoken: $%s
chatid: %s
message: %s
disable_web_page_preview: true
`, alertsTelegramTokenVar, strconv.Quote(t.ChatID), message)
}
if cfg.Discord {
if creds.DiscordWebhookURL == "" {
missing = append(missing, "discordWebhookUrl")
}
secrets[alertsDiscordURLVar] = creds.DiscordWebhookURL
fmt.Fprintf(&receivers, ` - uid: urbex-discord
type: discord
settings:
url: $%s
message: %s
`, alertsDiscordURLVar, message)
}
if cfg.Webhook {
if creds.AlertsWebhookURL == "" {
missing = append(missing, "alertsWebhookUrl")
}
secrets[alertsWebhookURLVar] = creds.AlertsWebhookURL
fmt.Fprintf(&receivers, ` - uid: urbex-webhook
type: webhook
settings:
url: $%s
`, alertsWebhookURLVar)
}
if len(missing) > 0 {
return "", nil, fmt.Errorf("observability.alerts needs the credential(s) %s: store them with 'urbex-op set <key>' (see docs/reference/credentials.md in urbforge/urbex)", strings.Join(missing, ", "))
}
var b strings.Builder
b.WriteString(`# Generated by urbex bootstrap from observability.alerts in
# urbex.platform.yaml (urbforge/urbex-cli#95) - do not edit. Sends the
# alerts of this Grafana to the channels configured there; their secrets
# come from the Stack's encrypted secrets, through Grafana's environment.
apiVersion: 1
contactPoints:
- orgId: 1
name: urbex
receivers:
`)
b.WriteString(receivers.String())
// The whole policy tree: this Grafana is the platform's, and its
// alerts are Urbex's (Grafana 13 has no default contact point to
// route the others to).
fmt.Fprintf(&b, `policies:
- orgId: 1
receiver: urbex
group_by: [alertname, service, host]
group_wait: %s
group_interval: %s
repeat_interval: %s
`, alertsGroupWait, alertsGroupInterval, alertsRepeatInterval)
return b.String(), secrets, nil
}
78 changes: 78 additions & 0 deletions internal/bootstrap/alerts_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package bootstrap

import (
"strings"
"testing"

"gopkg.in/yaml.v3"

"github.com/urbforge/urbex-cli/internal/credentials"
"github.com/urbforge/urbex-cli/internal/platform"
)

func TestAlertsProvisioning_NoneConfigured(t *testing.T) {
file, secrets, err := AlertsProvisioning(platform.AlertsConfig{}, &credentials.Credentials{})
if file != "" || secrets != nil || err != nil {
t.Fatalf("got %q %v %v, want nothing", file, secrets, err)
}
}

func TestAlertsProvisioning_MissingCredentials(t *testing.T) {
cfg := platform.AlertsConfig{Telegram: &platform.TelegramAlerts{ChatID: "-100123"}, Webhook: true}
_, _, err := AlertsProvisioning(cfg, &credentials.Credentials{})
if err == nil || !strings.Contains(err.Error(), "telegramBotToken, alertsWebhookUrl") {
t.Fatalf("err = %v, want both credentials named", err)
}
}

func TestAlertsProvisioning_AllChannels(t *testing.T) {
cfg := platform.AlertsConfig{Telegram: &platform.TelegramAlerts{ChatID: "-100123"}, Discord: true, Webhook: true}
creds := &credentials.Credentials{TelegramBotToken: "123:secret-token", DiscordWebhookURL: "https://discord.example.com/api/webhooks/1/secret", AlertsWebhookURL: "https://hooks.example.com/secret"}
file, secrets, err := AlertsProvisioning(cfg, creds)
if err != nil {
t.Fatal(err)
}
for _, v := range []string{creds.TelegramBotToken, creds.DiscordWebhookURL, creds.AlertsWebhookURL} {
if strings.Contains(file, v) {
t.Errorf("secret %q in clear in the provisioning file", v)
}
}
if secrets[alertsTelegramTokenVar] != creds.TelegramBotToken || secrets[alertsDiscordURLVar] != creds.DiscordWebhookURL || secrets[alertsWebhookURLVar] != creds.AlertsWebhookURL {
t.Errorf("secrets = %v", secrets)
}
var doc struct {
APIVersion int `yaml:"apiVersion"`
ContactPoints []struct {
Name string
Receivers []struct {
UID, Type string
Settings map[string]any
}
} `yaml:"contactPoints"`
Policies []struct {
Receiver string
GroupBy []string `yaml:"group_by"`
RepeatInterval string `yaml:"repeat_interval"`
}
}
if err := yaml.Unmarshal([]byte(file), &doc); err != nil {
t.Fatalf("not YAML: %v\n%s", err, file)
}
if doc.APIVersion != 1 || strings.Contains(file, "templates:") {
t.Errorf("apiVersion %d, or a named template (loaded late by Grafana)", doc.APIVersion)
}
if len(doc.ContactPoints) != 1 || len(doc.ContactPoints[0].Receivers) != 3 {
t.Fatalf("contact points: %+v", doc.ContactPoints)
}
tg := doc.ContactPoints[0].Receivers[0]
if tg.Type != "telegram" || tg.Settings["bottoken"] != "$"+alertsTelegramTokenVar || tg.Settings["chatid"] != "-100123" || tg.Settings["message"] != alertsMessage {
t.Errorf("telegram receiver: %+v", tg)
}
if doc.ContactPoints[0].Receivers[1].Settings["url"] != "$"+alertsDiscordURLVar {
t.Errorf("discord receiver: %+v", doc.ContactPoints[0].Receivers[1])
}
r := doc.Policies[0]
if r.Receiver != "urbex" || r.RepeatInterval != "4h" || strings.Join(r.GroupBy, ",") != "alertname,service,host" {
t.Errorf("route: %+v", r)
}
}
5 changes: 4 additions & 1 deletion internal/bootstrap/orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,10 @@ func (o *Orchestrator) Run(ctx context.Context, opts Options) error {
// The platform services Komodo deploys from the GitOps repo: their
// folders go in before setupPlatform pushes it.
o.Progress.Begin(PhaseGitOps)
stacks := o.platformStacks(tunnelToken)
stacks, err := o.platformStacks(tunnelToken)
if err != nil {
return err
}
if err := o.writePlatformStacks(ctx, opts.GitOpsRepoPath, stacks); err != nil {
return err
}
Expand Down
4 changes: 4 additions & 0 deletions internal/bootstrap/platform.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ type Endpoints struct {
TechnitiumURL string
// TechnitiumIP is Technitium's address: every LXC's first nameserver.
TechnitiumIP string
// Grafana is the observability stack's Grafana (http://host:3000).
Grafana string
}

// InternalEndpointsFor computes the base services' endpoints by internal
Expand All @@ -116,6 +118,7 @@ func InternalEndpointsFor(cfg *platform.Config) (Endpoints, error) {
Registry: name("git") + ":3000",
KomodoURL: "http://" + name("komodo") + ":9120",
LokiPushURL: "http://" + name("loki") + ":3100/loki/api/v1/push",
Grafana: "http://" + name("grafana") + ":3000",
PrometheusWriteURL: "http://" + name("prometheus") + ":9090/api/v1/write",
KeycloakURL: "http://" + name("auth") + ":8080",
TechnitiumURL: "http://" + name("dns") + ":5380",
Expand All @@ -135,6 +138,7 @@ func EndpointsFor(cfg *platform.Config) (Endpoints, error) {
Registry: giteaIP + ":3000",
KomodoURL: "http://" + ips[Hostname("komodo")] + ":9120",
LokiPushURL: "http://" + ips[Hostname("observability")] + ":3100/loki/api/v1/push",
Grafana: "http://" + ips[Hostname("observability")] + ":3000",
PrometheusWriteURL: "http://" + ips[Hostname("observability")] + ":9090/api/v1/write",
KeycloakURL: "http://" + ips[Hostname("keycloak")] + ":8080",
TechnitiumURL: "http://" + ips[Hostname("technitium")] + ":5380",
Expand Down
67 changes: 63 additions & 4 deletions internal/bootstrap/stacks.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"strings"
"sync"

"github.com/urbforge/urbex-cli/internal/platform"
assets "github.com/urbforge/urbex-cli/platform"

"github.com/urbforge/urbex-cli/internal/komodo"
Expand All @@ -34,6 +35,10 @@ type PlatformStack struct {
Name, Dir string
Config map[string]string
Secrets map[string]string
// Generated are files urbex writes besides the embedded ones, by
// their path in the Stack's folder; a path mapped to "" is removed,
// so that a file no longer wanted doesn't stay behind.
Generated map[string]string
}

// ComposeWrapper is the command every Stack deployed from the GitOps repo
Expand All @@ -52,7 +57,7 @@ const PlatformStacksDir = "platform"
// platformStacks are the platform services deployed from the GitOps
// repo: Keycloak, the observability stack and, with Cloudflare, the
// tunnel, whose connector token is tunnelToken.
func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack {
func (o *Orchestrator) platformStacks(tunnelToken string) ([]PlatformStack, error) {
// Keycloak runs in production mode on its own PostgreSQL, in the same
// Stack (urbex-cli#36). Both containers read config.env and the
// secrets: KC_* configure Keycloak, POSTGRES_* the database.
Expand Down Expand Up @@ -89,14 +94,45 @@ func (o *Orchestrator) platformStacks(tunnelToken string) []PlatformStack {
"KC_DB_PASSWORD": o.Credentials.KeycloakDBPassword,
"POSTGRES_PASSWORD": o.Credentials.KeycloakDBPassword,
}},
{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{},
Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword}},
}
observability, err := o.observabilityStack()
if err != nil {
return nil, err
}
stacks = append(stacks, observability)
if o.Config.Cloudflare.Enabled() {
stacks = append(stacks, PlatformStack{Name: Hostname(TunnelService.Name), Dir: "tunnel", Config: map[string]string{},
Secrets: map[string]string{"TUNNEL_TOKEN": tunnelToken}})
}
return stacks
return stacks, nil
}

// observabilityStack is Prometheus, Loki and Grafana; with
// observability.alerts, Grafana also sends Urbex's alerts to its channels
// (urbforge/urbex-cli#95), and its links point to its own address.
func (o *Orchestrator) observabilityStack() (PlatformStack, error) {
st := PlatformStack{Name: Hostname("observability"), Dir: "observability", Config: map[string]string{},
Secrets: map[string]string{"GF_SECURITY_ADMIN_PASSWORD": o.Credentials.GrafanaAdminPassword},
Generated: map[string]string{AlertsProvisioningFile: ""}}
file, secrets, err := AlertsProvisioning(o.Config.Observability.Alerts, o.Credentials)
if err != nil || file == "" {
return st, err
}
st.Generated[AlertsProvisioningFile] = file
for k, v := range secrets {
st.Secrets[k] = v
}
st.Config["GF_SERVER_ROOT_URL"] = GrafanaURL(o.Config)
return st, nil
}

// GrafanaURL is the address the alerts link to: Grafana behind Access with
// Cloudflare, else its internal name, on the LAN or the VPN.
func GrafanaURL(cfg *platform.Config) string {
if host := PublicEndpointsFor(cfg).Grafana; host != "" {
return "https://" + host + "/"
}
return "http://" + cfg.InternalName("grafana") + ":3000/"
}

// Keycloak's database and its owner, in the Keycloak Stack's PostgreSQL.
Expand Down Expand Up @@ -144,6 +180,29 @@ func (o *Orchestrator) writePlatformStacks(ctx context.Context, gitopsRepoPath s
if err != nil {
return fmt.Errorf("writing the %s stack: %w", st.Dir, err)
}
generated := make([]string, 0, len(st.Generated))
for p := range st.Generated {
generated = append(generated, p)
}
sort.Strings(generated)
for _, p := range generated {
dest := filepath.Join(dir, filepath.FromSlash(p))
data := st.Generated[p]
if data == "" {
if err := os.Remove(dest); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
continue
}
fmt.Fprintf(files, "%s %d\n", p, len(data))
files.Write([]byte(data))
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err
}
if err := os.WriteFile(dest, []byte(data), 0o644); err != nil {
return err
}
}
// The files' hash in config.env, which the containers that read
// the files also read: compose recreates them when a file
// changes. A container keeps a single bind-mounted file as it
Expand Down
Loading