From a60a7d50356efa099c8c4a64a51e237112495255 Mon Sep 17 00:00:00 2001 From: Elio Tohme Date: Tue, 18 Aug 2026 09:39:32 +0200 Subject: [PATCH] feat: add host whitelist configmap --- README.md | 6 + charts/cloudflare-exporter/README.md | 3 + .../ci/assert-host-whitelist-render.sh | 32 + .../ci/host-whitelist-values.yaml | 2 + .../templates/deployment.yaml | 18 + charts/cloudflare-exporter/values.schema.json | 15 + charts/cloudflare-exporter/values.yaml | 2 + examples/host-whitelist-configmap.yaml | 9 + go.mod | 3 +- host_whitelist.go | 202 +++++++ host_whitelist_test.go | 547 ++++++++++++++++++ main.go | 77 ++- prometheus.go | 189 +++--- 13 files changed, 1001 insertions(+), 104 deletions(-) create mode 100755 charts/cloudflare-exporter/ci/assert-host-whitelist-render.sh create mode 100644 charts/cloudflare-exporter/ci/host-whitelist-values.yaml create mode 100644 charts/cloudflare-exporter/values.schema.json create mode 100644 examples/host-whitelist-configmap.yaml create mode 100644 host_whitelist.go create mode 100644 host_whitelist_test.go diff --git a/README.md b/README.md index fbf5001f..41d8d829 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ The exporter can be configured using env variables or command flags. | `ZONE_` | `DEPRECATED since 0.0.5` (optional) Zone ID. Add zones you want to scrape by adding env vars in this format. You can find the zone ids in Cloudflare dashboards. | | `LOG_LEVEL` | Set loglevel. Options are error, warn, info, debug. default `error` | +### Host whitelist + +Set the chart value `hostWhitelist.configMapName` to an externally managed ConfigMap in the release namespace. The chart mounts only its `hosts.yaml` key read-only at `/etc/cloudflare-exporter/hosts.yaml`; it never creates the ConfigMap. See `examples/host-whitelist-configmap.yaml`. + +The file is reread before each scrape. It must be exactly one mapping with one `hosts` list of strings. YAML and equivalent JSON are accepted; duplicate keys, merge keys, scalar/null or multi-document input, extra keys, and invalid entries are rejected. Matching is exact and unnormalized. Before any valid whitelist is loaded, a missing or unmounted ConfigMap/file scrapes all hosts. A valid `hosts: []` also explicitly scrapes all hosts. Invalid updates retain the last valid list; after a valid non-empty whitelist has loaded, an unreadable or missing file retains that last valid whitelist. Covered paths are request analytics, firewall host analytics, colocation analytics, and edge-errors-by-path; non-host metrics are unchanged. + Corresponding flags: ``` diff --git a/charts/cloudflare-exporter/README.md b/charts/cloudflare-exporter/README.md index 2f53e57a..9bb9842e 100644 --- a/charts/cloudflare-exporter/README.md +++ b/charts/cloudflare-exporter/README.md @@ -23,6 +23,7 @@ The following table lists the configurable parameters of the Cloudflare-exporter | `image.tag` | | `"0.0.2"` | | `env` | | `[]` | | `secretRef` | The name of a secret with environment variables | `""` | +| `hostWhitelist.configMapName` | External ConfigMap containing the `hosts.yaml` whitelist key | `""` | | `imagePullSecrets` | | `[]` | | `nameOverride` | | `""` | | `fullnameOverride` | | `""` | @@ -50,6 +51,8 @@ The following table lists the configurable parameters of the Cloudflare-exporter | `tolerations` | | `[]` | | `affinity` | | `{}` | +When configured, the operator must create `hostWhitelist.configMapName` in the release namespace. The chart projects only `hosts.yaml` read-only into `/etc/cloudflare-exporter`; it does not create or own a ConfigMap. The runtime default path is `/etc/cloudflare-exporter/hosts.yaml`. Before any valid whitelist is loaded, a missing or unmounted ConfigMap means scrape all hosts. A valid `hosts: []` also means scrape all hosts. After a valid non-empty whitelist is loaded, an unreadable or missing file retains the last valid whitelist. See `examples/host-whitelist-configmap.yaml`, `ci/host-whitelist-values.yaml`, and `ci/assert-host-whitelist-render.sh`. + ## Contributing and reporting issues diff --git a/charts/cloudflare-exporter/ci/assert-host-whitelist-render.sh b/charts/cloudflare-exporter/ci/assert-host-whitelist-render.sh new file mode 100755 index 00000000..cc993648 --- /dev/null +++ b/charts/cloudflare-exporter/ci/assert-host-whitelist-render.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +chart_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +helm template test "$chart_dir" >"$tmp_dir/default.yaml" +helm template test "$chart_dir" -f "$chart_dir/ci/host-whitelist-values.yaml" >"$tmp_dir/enabled.yaml" + +count() { grep -cF -- "$1" "$2" || true; } +assert_count() { + [ "$(count "$1" "$2")" -eq "$3" ] || { + echo "expected $3 occurrences of $1 in $2" >&2 + exit 1 + } +} + +assert_count 'name: host-whitelist' "$tmp_dir/default.yaml" 0 +assert_count '/etc/cloudflare-exporter/hosts.yaml' "$tmp_dir/default.yaml" 0 +assert_count 'configMap:' "$tmp_dir/default.yaml" 0 +assert_count 'name: host-whitelist' "$tmp_dir/enabled.yaml" 2 +assert_count 'volumes:' "$tmp_dir/enabled.yaml" 1 +assert_count 'volumeMounts:' "$tmp_dir/enabled.yaml" 1 +assert_count 'mountPath: /etc/cloudflare-exporter' "$tmp_dir/enabled.yaml" 1 +assert_count 'readOnly: true' "$tmp_dir/enabled.yaml" 1 +assert_count 'configMap:' "$tmp_dir/enabled.yaml" 1 +assert_count 'name: cloudflare-exporter-hosts' "$tmp_dir/enabled.yaml" 1 +assert_count 'key: hosts.yaml' "$tmp_dir/enabled.yaml" 1 +assert_count 'path: hosts.yaml' "$tmp_dir/enabled.yaml" 1 +assert_count 'mode: 0444' "$tmp_dir/enabled.yaml" 1 +assert_count 'kind: ConfigMap' "$tmp_dir/enabled.yaml" 0 diff --git a/charts/cloudflare-exporter/ci/host-whitelist-values.yaml b/charts/cloudflare-exporter/ci/host-whitelist-values.yaml new file mode 100644 index 00000000..15a812fa --- /dev/null +++ b/charts/cloudflare-exporter/ci/host-whitelist-values.yaml @@ -0,0 +1,2 @@ +hostWhitelist: + configMapName: cloudflare-exporter-hosts diff --git a/charts/cloudflare-exporter/templates/deployment.yaml b/charts/cloudflare-exporter/templates/deployment.yaml index 5d569745..22fb5c9c 100644 --- a/charts/cloudflare-exporter/templates/deployment.yaml +++ b/charts/cloudflare-exporter/templates/deployment.yaml @@ -31,6 +31,18 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} serviceAccountName: {{ include "cloudflare-exporter.serviceAccountName" . }} + {{- if .Values.hostWhitelist.configMapName }} + volumes: + - name: host-whitelist + projected: + sources: + - configMap: + name: {{ .Values.hostWhitelist.configMapName }} + items: + - key: hosts.yaml + path: hosts.yaml + mode: 0444 + {{- end }} containers: - name: {{ .Chart.Name }} securityContext: @@ -45,6 +57,12 @@ spec: {{- toYaml .Values.resources | nindent 12 }} env: {{- toYaml .Values.env | nindent 12 }} + {{- if .Values.hostWhitelist.configMapName }} + volumeMounts: + - name: host-whitelist + mountPath: /etc/cloudflare-exporter + readOnly: true + {{- end }} {{- if .Values.secretRef }} envFrom: - secretRef: diff --git a/charts/cloudflare-exporter/values.schema.json b/charts/cloudflare-exporter/values.schema.json new file mode 100644 index 00000000..1751510b --- /dev/null +++ b/charts/cloudflare-exporter/values.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "hostWhitelist": { + "type": "object", + "properties": { + "configMapName": { "type": "string" } + }, + "required": ["configMapName"], + "additionalProperties": false + } + }, + "required": ["hostWhitelist"] +} diff --git a/charts/cloudflare-exporter/values.yaml b/charts/cloudflare-exporter/values.yaml index 4a0e7954..9615c144 100644 --- a/charts/cloudflare-exporter/values.yaml +++ b/charts/cloudflare-exporter/values.yaml @@ -11,6 +11,8 @@ image: # tag: latest env: [] secretRef: "" +hostWhitelist: + configMapName: "" imagePullSecrets: [] nameOverride: "" fullnameOverride: "" diff --git a/examples/host-whitelist-configmap.yaml b/examples/host-whitelist-configmap.yaml new file mode 100644 index 00000000..374129e9 --- /dev/null +++ b/examples/host-whitelist-configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cloudflare-exporter-hosts +data: + hosts.yaml: | + hosts: + - www.example.com + - api.example.com diff --git a/go.mod b/go.mod index 0dc90506..64975a66 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,13 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -46,5 +48,4 @@ require ( golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/host_whitelist.go b/host_whitelist.go new file mode 100644 index 00000000..d75fbe4d --- /dev/null +++ b/host_whitelist.go @@ -0,0 +1,202 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/yaml.v3" +) + +const hostWhitelistPath = "/etc/cloudflare-exporter/hosts.yaml" + +type hostWhitelist struct { + enabled bool + allowed map[string]struct{} +} + +func emptyHostWhitelist() hostWhitelist { + return hostWhitelist{allowed: map[string]struct{}{}} +} + +func (w hostWhitelist) Allows(host string) bool { + if !w.enabled || len(w.allowed) == 0 { + return true + } + _, ok := w.allowed[host] + return ok +} + +func readHostWhitelist(path string) (hostWhitelist, string, error) { + b, err := os.ReadFile(path) + if err != nil { + return emptyHostWhitelist(), "missing/unreadable", err + } + if len(b) == 0 { + return emptyHostWhitelist(), "empty", errors.New("empty whitelist") + } + + decoder := yaml.NewDecoder(bytes.NewReader(b)) + var document yaml.Node + if err := decoder.Decode(&document); err != nil { + return emptyHostWhitelist(), "malformed", errors.New("malformed whitelist") + } + var extra yaml.Node + if err := decoder.Decode(&extra); err != io.EOF { + return emptyHostWhitelist(), "malformed", errors.New("multiple whitelist documents") + } + + if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return emptyHostWhitelist(), "invalid schema", errors.New("whitelist must be one mapping") + } + root := document.Content[0] + if len(root.Content) != 2 { + return emptyHostWhitelist(), "invalid schema", errors.New("whitelist must contain only hosts") + } + var hosts *yaml.Node + for i := 0; i < len(root.Content); i += 2 { + key, value := root.Content[i], root.Content[i+1] + if key.Value == "<<" { + return emptyHostWhitelist(), "invalid schema", errors.New("merge keys are not allowed") + } + if key.Value != "hosts" || hosts != nil { + return emptyHostWhitelist(), "invalid schema", errors.New("invalid whitelist key") + } + hosts = value + } + if hosts == nil || hosts.Kind != yaml.SequenceNode { + return emptyHostWhitelist(), "invalid schema", errors.New("hosts must be a list") + } + + allowed := make(map[string]struct{}, len(hosts.Content)) + for _, host := range hosts.Content { + if host.Kind != yaml.ScalarNode || host.Tag != "!!str" { + return emptyHostWhitelist(), "invalid schema", errors.New("hosts must contain strings") + } + allowed[host.Value] = struct{}{} + } + return hostWhitelist{enabled: true, allowed: allowed}, "", nil +} + +func loadHostWhitelist(path string, previous hostWhitelist) hostWhitelist { + if path == "" { + return emptyHostWhitelist() + } + next, category, err := readHostWhitelist(path) + if err != nil { + log.WithField("path", path).WithField("category", category).Warn("host whitelist reload failed") + if previous.enabled { + return previous + } + return emptyHostWhitelist() + } + if !sameHostWhitelist(previous, next) { + log.WithField("path", path).WithField("hosts", len(next.allowed)).Info("host whitelist replaced") + } + return next +} + +func sameHostWhitelist(a, b hostWhitelist) bool { + if a.enabled != b.enabled || len(a.allowed) != len(b.allowed) { + return false + } + for host := range a.allowed { + if _, ok := b.allowed[host]; !ok { + return false + } + } + return true +} + +type hostSeriesVector interface { + DeleteLabelValues(labelValues ...string) bool +} + +type hostSeriesFamily struct { + vector hostSeriesVector + labels []string + emitted map[string]map[string]struct{} + observed map[string]map[string]struct{} +} + +type hostSeriesRegistry struct { + mu sync.Mutex + families []hostSeriesFamily +} + +func newHostSeriesRegistry() *hostSeriesRegistry { return &hostSeriesRegistry{} } + +var hostSeriesRegistryState = newHostSeriesRegistry() + +func (r *hostSeriesRegistry) Register(vector hostSeriesVector, labels ...string) { + r.mu.Lock() + defer r.mu.Unlock() + r.families = append(r.families, hostSeriesFamily{ + vector: vector, + labels: labels, + emitted: map[string]map[string]struct{}{}, + observed: map[string]map[string]struct{}{}, + }) +} + +func (r *hostSeriesRegistry) Observe(vector hostSeriesVector, labels prometheus.Labels) { + r.mu.Lock() + defer r.mu.Unlock() + for i := range r.families { + if r.families[i].vector != vector { + continue + } + values := make([]string, len(r.families[i].labels)) + for j, label := range r.families[i].labels { + values[j] = labels[label] + } + host := labels["host"] + key, _ := json.Marshal(values) + if r.families[i].observed[host] == nil { + r.families[i].observed[host] = map[string]struct{}{} + } + r.families[i].observed[host][string(key)] = struct{}{} + return + } +} + +func deleteHostSeriesTuples(family *hostSeriesFamily, tuples map[string]struct{}) { + for key := range tuples { + var values []string + if err := json.Unmarshal([]byte(key), &values); err == nil { + family.vector.DeleteLabelValues(values...) + } + } +} + +func (r *hostSeriesRegistry) ResetScrape() { + r.mu.Lock() + defer r.mu.Unlock() + for i := range r.families { + r.families[i].observed = map[string]map[string]struct{}{} + } +} + +func (r *hostSeriesRegistry) Reconcile() { + r.mu.Lock() + defer r.mu.Unlock() + for i := range r.families { + family := &r.families[i] + for host, emitted := range family.emitted { + observed := family.observed[host] + stale := make(map[string]struct{}) + for key := range emitted { + if _, ok := observed[key]; !ok { + stale[key] = struct{}{} + } + } + deleteHostSeriesTuples(family, stale) + } + family.emitted = family.observed + family.observed = map[string]map[string]struct{}{} + } +} diff --git a/host_whitelist_test.go b/host_whitelist_test.go new file mode 100644 index 00000000..7be72b2a --- /dev/null +++ b/host_whitelist_test.go @@ -0,0 +1,547 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/spf13/viper" +) + +func writeWhitelist(t *testing.T, input string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "hosts.yaml") + if err := os.WriteFile(path, []byte(input), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func gatheredHostTuples(t *testing.T, collector prometheus.Collector, labels []string) map[string]struct{} { + t.Helper() + registry := prometheus.NewPedanticRegistry() + if err := registry.Register(collector); err != nil { + t.Fatal(err) + } + families, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + got := make(map[string]struct{}) + for _, family := range families { + for _, metric := range family.GetMetric() { + values := make([]string, len(labels)) + for i, name := range labels { + for _, label := range metric.GetLabel() { + if label.GetName() == name { + values[i] = label.GetValue() + break + } + } + } + key, err := json.Marshal(values) + if err != nil { + t.Fatal(err) + } + got[string(key)] = struct{}{} + } + } + return got +} + +func TestReadHostWhitelist(t *testing.T) { + tests := []struct { + name string + input string + want []string + }{ + {name: "yaml", input: "hosts:\n - www.example.com\n - api.example.com\n", want: []string{"www.example.com", "api.example.com"}}, + {name: "json", input: `{"hosts":["www.example.com","api.example.com"]}`, want: []string{"www.example.com", "api.example.com"}}, + {name: "empty valid list", input: "hosts: []\n", want: []string{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, category, err := readHostWhitelist(writeWhitelist(t, tt.input)) + if err != nil || category != "" { + t.Fatalf("readHostWhitelist() error = %v, category = %q", err, category) + } + if len(got.allowed) != len(tt.want) { + t.Fatalf("got %d hosts, want %d", len(got.allowed), len(tt.want)) + } + for _, host := range tt.want { + if _, ok := got.allowed[host]; !ok { + t.Errorf("host %q not allowed", host) + } + } + if !got.enabled { + t.Error("valid file must enable filtering") + } + }) + } +} + +func TestReadHostWhitelistRejectsInvalidDocuments(t *testing.T) { + tests := []struct { + name string + input string + category string + }{ + {name: "duplicate key", input: "hosts: [www.example.com]\nhosts: [api.example.com]", category: "invalid schema"}, + {name: "scalar", input: "42", category: "invalid schema"}, + {name: "null", input: "null", category: "invalid schema"}, + {name: "merge key", input: "base: &base {hosts: [www.example.com]}\n<<: *base", category: "invalid schema"}, + {name: "multiple documents", input: "hosts: [www.example.com]\n---\nhosts: [api.example.com]", category: "malformed"}, + {name: "additional key", input: "hosts: [www.example.com]\nother: value", category: "invalid schema"}, + {name: "non-list hosts", input: "hosts: www.example.com", category: "invalid schema"}, + {name: "non-string entry", input: "hosts: [www.example.com, 42]", category: "invalid schema"}, + {name: "null entry", input: "hosts: [null]", category: "invalid schema"}, + {name: "empty", input: "", category: "empty"}, + {name: "malformed", input: "hosts: [", category: "malformed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, category, err := readHostWhitelist(writeWhitelist(t, tt.input)) + if err == nil || category != tt.category { + t.Fatalf("readHostWhitelist() error = %v, category = %q, want %q", err, category, tt.category) + } + }) + } +} + +func TestLoadHostWhitelistRetainsPreviousSnapshot(t *testing.T) { + path := writeWhitelist(t, "hosts: [good.example.com]") + previous := loadHostWhitelist(path, emptyHostWhitelist()) + if _, ok := previous.allowed["good.example.com"]; !previous.enabled || !ok { + t.Fatalf("initial load = %#v", previous) + } + + for _, input := range []string{"", "hosts: [", "hosts: www.example.com", "hosts: [bad.example.com]\nother: value"} { + if err := os.WriteFile(path, []byte(input), 0o600); err != nil { + t.Fatal(err) + } + got := loadHostWhitelist(path, previous) + if _, ok := got.allowed["good.example.com"]; !got.enabled || !ok || len(got.allowed) != 1 { + t.Fatalf("input %q changed snapshot to %#v", input, got) + } + } +} + +func TestLoadHostWhitelistReplacesValidSnapshot(t *testing.T) { + path := writeWhitelist(t, "hosts: [good.example.com]") + previous := loadHostWhitelist(path, emptyHostWhitelist()) + if err := os.WriteFile(path, []byte("hosts: [replacement.example.com]"), 0o600); err != nil { + t.Fatal(err) + } + + got := loadHostWhitelist(path, previous) + if !got.enabled || !got.Allows("replacement.example.com") || got.Allows("good.example.com") { + t.Fatalf("replacement snapshot = %#v", got) + } +} + +func TestLoadHostWhitelistRetainsValidSnapshotWhenFileBecomesUnreadable(t *testing.T) { + path := writeWhitelist(t, "hosts: [good.example.com]") + previous := loadHostWhitelist(path, emptyHostWhitelist()) + if err := os.WriteFile(path, []byte("hosts: ["), 0o600); err != nil { + t.Fatal(err) + } + got := loadHostWhitelist(path, previous) + if !sameHostWhitelist(got, previous) { + t.Fatalf("malformed file changed snapshot to %#v", got) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + + got = loadHostWhitelist(path, previous) + if !sameHostWhitelist(got, previous) || !got.enabled || !got.Allows("good.example.com") || got.Allows("bad.example.com") { + t.Fatalf("unreadable file changed snapshot to %#v", got) + } +} + +func TestLoadHostWhitelistDisabledStates(t *testing.T) { + if got := loadHostWhitelist("", emptyHostWhitelist()); got.enabled || !got.Allows("arbitrary.example.com") { + t.Fatalf("empty path = %#v", got) + } + if got := loadHostWhitelist(filepath.Join(t.TempDir(), "missing"), emptyHostWhitelist()); got.enabled || !got.Allows("arbitrary.example.com") { + t.Fatalf("missing initial path = %#v", got) + } +} + +func TestLoadHostWhitelistEmptyListRetainsScrapeAll(t *testing.T) { + path := writeWhitelist(t, "hosts: []") + previous := loadHostWhitelist(path, emptyHostWhitelist()) + if !previous.enabled || len(previous.allowed) != 0 || !previous.Allows("anything.example.com") { + t.Fatalf("empty valid snapshot = %#v", previous) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + got := loadHostWhitelist(path, previous) + if !sameHostWhitelist(got, previous) || !got.Allows("anything.example.com") { + t.Fatalf("retained empty snapshot = %#v", got) + } +} + +func TestHostWhitelistAllowsExactMatches(t *testing.T) { + path := writeWhitelist(t, "hosts: [www.example.com]") + whitelist, _, err := readHostWhitelist(path) + if err != nil { + t.Fatal(err) + } + for _, host := range []string{"WWW.EXAMPLE.COM", "example.com", "www.example.com.evil", "evilwww.example.com", "*.example.com", " www.example.com", "www.example.com ", "unrelated.example.com"} { + if whitelist.Allows(host) { + t.Errorf("Allows(%q) = true", host) + } + } + if !whitelist.Allows("www.example.com") { + t.Error("exact host was rejected") + } + if !emptyHostWhitelist().Allows("anything.example.com") { + t.Error("disabled whitelist filtered a host") + } + if got, _, err := readHostWhitelist(writeWhitelist(t, "hosts: []")); err != nil || !got.Allows("anything.example.com") { + t.Error("enabled empty whitelist did not allow all hosts") + } +} + +func TestHostWhitelistSnapshotIsShared(t *testing.T) { + snapshot := hostWhitelist{enabled: true, allowed: map[string]struct{}{"good.example.com": {}}} + start := make(chan struct{}) + results := make(chan bool, 4) + for i := 0; i < 4; i++ { + go func() { + <-start + results <- snapshot.Allows("good.example.com") && !snapshot.Allows("bad.example.com") + }() + } + close(start) + for i := 0; i < 4; i++ { + if !<-results { + t.Fatal("collector did not observe the same snapshot") + } + } +} + +func TestHostSeriesRegistryReconcilesRemovedTuples(t *testing.T) { + registry := newHostSeriesRegistry() + counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "test_host_counter"}, []string{"host", "kind"}) + registry.Register(counter, "host", "kind") + for _, kind := range []string{"kept", "stale"} { + counter.WithLabelValues("host.example.com", kind).Add(1) + registry.Observe(counter, prometheus.Labels{"host": "host.example.com", "kind": kind}) + } + registry.Reconcile() + registry.ResetScrape() + counter.WithLabelValues("host.example.com", "kept").Add(1) + registry.Observe(counter, prometheus.Labels{"host": "host.example.com", "kind": "kept"}) + registry.Reconcile() + if got := gatheredHostTuples(t, counter, []string{"host", "kind"}); len(got) != 1 { + t.Fatalf("gathered tuples = %#v", got) + } + if counter.DeleteLabelValues("host.example.com", "stale") { + t.Fatal("stale tuple was not deleted") + } +} + +func TestHostSeriesRegistryReconcilesCounterAndGaugeWithDifferentLabelOrders(t *testing.T) { + registry := newHostSeriesRegistry() + counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "test_tuple_counter"}, []string{"host", "kind"}) + gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_tuple_gauge"}, []string{"kind", "host"}) + registry.Register(counter, "host", "kind") + registry.Register(gauge, "kind", "host") + counter.WithLabelValues("stale.example.com", "counter").Add(1) + registry.Observe(counter, prometheus.Labels{"host": "stale.example.com", "kind": "counter"}) + gauge.WithLabelValues("gauge", "stale.example.com").Set(1) + registry.Observe(gauge, prometheus.Labels{"kind": "gauge", "host": "stale.example.com"}) + registry.Reconcile() + registry.ResetScrape() + registry.Reconcile() + if len(gatheredHostTuples(t, counter, []string{"host", "kind"})) != 0 || len(gatheredHostTuples(t, gauge, []string{"kind", "host"})) != 0 { + t.Fatal("stale series was not removed") + } +} + +func TestHostSeriesRegistryResetKeepsFamiliesAndBoundsBookkeeping(t *testing.T) { + registry := newHostSeriesRegistry() + vector := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "test_reset_host_counter"}, []string{"host"}) + registry.Register(vector, "host") + + vector.WithLabelValues("first.example.com").Add(1) + registry.Observe(vector, prometheus.Labels{"host": "first.example.com"}) + registry.Reconcile() + registry.ResetScrape() + if len(registry.families) != 1 || len(registry.families[0].emitted) != 1 || len(registry.families[0].observed) != 0 { + t.Fatalf("reset bookkeeping = %#v", registry.families) + } + registry.Reconcile() + if len(gatheredHostTuples(t, vector, []string{"host"})) != 0 { + t.Fatal("series was deleted from scrape-all whitelist") + } +} + +func TestFetchMetricsReconcilesZeroZoneAndErrorScrapes(t *testing.T) { + previousRegistry := hostSeriesRegistryState + previousSnapshot := hostWhitelistSnapshot + previousPath := viper.GetString("host_whitelist_path") + t.Cleanup(func() { + hostSeriesRegistryState = previousRegistry + hostWhitelistSnapshot = previousSnapshot + viper.Set("host_whitelist_path", previousPath) + }) + + vector := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_orchestration_host_gauge"}, []string{"zone", "host", "status"}) + hostSeriesRegistryState = newHostSeriesRegistry() + hostSeriesRegistryState.Register(vector, "zone", "host", "status") + viper.Set("host_whitelist_path", "") + + labels := prometheus.Labels{"zone": "zone-a", "host": "stale.example.com", "status": "200"} + vector.With(labels).Set(1) + hostSeriesRegistryState.Observe(vector, labels) + hostSeriesRegistryState.Reconcile() + + fetchMetricsWithRunner(func(hostWhitelist, *sync.WaitGroup) {}) + if got := gatheredHostTuples(t, vector, []string{"zone", "host", "status"}); len(got) != 0 { + t.Fatalf("zero-zone scrape retained tuples = %#v", got) + } + + vector.With(labels).Set(1) + hostSeriesRegistryState.Observe(vector, labels) + hostSeriesRegistryState.Reconcile() + fetchMetricsWithRunner(func(_ hostWhitelist, wg *sync.WaitGroup) { + wg.Add(1) + go func() { defer wg.Done() }() + }) + if got := gatheredHostTuples(t, vector, []string{"zone", "host", "status"}); len(got) != 0 { + t.Fatalf("error scrape retained tuples = %#v", got) + } +} + +func TestHostMetricFilteringBoundaries(t *testing.T) { + allowed := hostWhitelist{enabled: true, allowed: map[string]struct{}{"good.example.com": {}}} + registry := newHostSeriesRegistry() + registry.Register(zoneRequestOriginStatusCountryHost, "zone", "account", "status", "country", "host") + registry.Register(zoneRequestStatusCountryHost, "zone", "account", "status", "country", "host") + registry.Register(zoneFirewallEventsCount, "zone", "account", "action", "source", "rule", "host", "country") + registry.Register(zoneColocationVisits, "zone", "account", "colocation", "host") + registry.Register(zoneColocationEdgeResponseBytes, "zone", "account", "colocation", "host") + registry.Register(zoneColocationRequestsTotal, "zone", "account", "colocation", "host") + registry.Register(zoneEdgeErrorsByPath, "zone", "account", "status", "host", "path") + + zone := zoneResp{} + if err := json.Unmarshal([]byte(`{"httpRequestsAdaptiveGroups":[{"count":1,"dimensions":{"originResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"good.example.com"}},{"count":1,"dimensions":{"originResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"bad.example.com"}}],"httpRequestsEdgeCountryHost":[{"count":1,"dimensions":{"edgeResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"good.example.com"}},{"count":1,"dimensions":{"edgeResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"bad.example.com"}}],"firewallEventsAdaptiveGroups":[{"count":1,"dimensions":{"action":"block","source":"waf","ruleId":"rule","clientCountryName":"US","clientRequestHTTPHost":"good.example.com"}},{"count":1,"dimensions":{"action":"block","source":"waf","ruleId":"rule","clientCountryName":"US","clientRequestHTTPHost":"bad.example.com"}}]}`), &zone); err != nil { + t.Fatal(err) + } + addHTTPAdaptiveGroups(&zone, "zone", "account", allowed, registry) + addFirewallGroupsWithRules(&zone, "zone", "account", allowed, registry, map[string]string{"rule": "rule"}) + + colo := zoneRespColo{} + if err := json.Unmarshal([]byte(`{"httpRequestsAdaptiveGroups":[{"count":1,"dimensions":{"coloCode":"AMS","clientRequestHTTPHost":"good.example.com"}},{"count":1,"dimensions":{"coloCode":"AMS","clientRequestHTTPHost":"bad.example.com"}}]}`), &colo); err != nil { + t.Fatal(err) + } + for _, c := range colo.ColoGroups { + addColocationGroup(c, "zone", "account", allowed, registry) + } + + edges := zoneRespEdgeErrorsByPath{} + if err := json.Unmarshal([]byte(`{"httpRequestsAdaptiveGroups":[{"count":1,"dimensions":{"edgeResponseStatus":500,"clientRequestHTTPHost":"good.example.com","clientRequestPath":"/users/1"}},{"count":1,"dimensions":{"edgeResponseStatus":500,"clientRequestHTTPHost":"bad.example.com","clientRequestPath":"/users/2"}}]}`), &edges); err != nil { + t.Fatal(err) + } + addEdgeErrorsByPath(&edges, "zone", "account", allowed, registry) + + registry.mu.Lock() + defer registry.mu.Unlock() + for _, family := range registry.families { + if len(family.observed) != 1 { + t.Errorf("observed hosts = %#v", family.observed) + } + if _, ok := family.observed["good.example.com"]; !ok { + t.Errorf("allowed host missing: %#v", family.observed) + } + } + + nonHost := zoneResp{} + if err := json.Unmarshal([]byte(`{"httpRequests1mGroups":[{"sum":{"requests":7}}]}`), &nonHost); err != nil { + t.Fatal(err) + } + addHTTPGroups(&nonHost, "non-host-zone", "account") + if got := testutil.ToFloat64(zoneRequestTotal.WithLabelValues("non-host-zone", "account")); got != 7 { + t.Fatalf("non-host request total = %v", got) + } +} + +func task3Registry() *hostSeriesRegistry { + registry := newHostSeriesRegistry() + registry.Register(zoneRequestOriginStatusCountryHost, "zone", "account", "status", "country", "host") + registry.Register(zoneRequestOriginStatusCountryHostP50, "zone", "account", "status", "country", "host") + registry.Register(zoneRequestOriginStatusCountryHostP95, "zone", "account", "status", "country", "host") + registry.Register(zoneRequestOriginStatusCountryHostP99, "zone", "account", "status", "country", "host") + registry.Register(zoneRequestStatusCountryHost, "zone", "account", "status", "country", "host") + registry.Register(zoneColocationVisits, "zone", "account", "colocation", "host") + registry.Register(zoneColocationEdgeResponseBytes, "zone", "account", "colocation", "host") + registry.Register(zoneColocationRequestsTotal, "zone", "account", "colocation", "host") + registry.Register(zoneFirewallEventsCount, "zone", "account", "action", "source", "rule", "host", "country") + registry.Register(zoneEdgeErrorsByPath, "zone", "account", "status", "host", "path") + return registry +} + +func populateTask3Families(t *testing.T, zone string, snapshot hostWhitelist, registry *hostSeriesRegistry) { + t.Helper() + const hosts = `{"count":%d,"dimensions":{"originResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"%s"},"quantiles":{"originResponseDurationMsP50":101,"originResponseDurationMsP95":202,"originResponseDurationMsP99":303}}` + var zoneData zoneResp + if err := json.Unmarshal([]byte(fmt.Sprintf(`{"httpRequestsAdaptiveGroups":[`+hosts+`,`+hosts+`],"httpRequestsEdgeCountryHost":[{"count":12,"dimensions":{"edgeResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"good.example.com"}},{"count":12,"dimensions":{"edgeResponseStatus":200,"clientCountryName":"US","clientRequestHTTPHost":"bad.example.com"}}],"firewallEventsAdaptiveGroups":[{"count":13,"dimensions":{"action":"block","source":"waf","ruleId":"rule-1","clientCountryName":"US","clientRequestHTTPHost":"good.example.com"}},{"count":13,"dimensions":{"action":"block","source":"waf","ruleId":"rule-1","clientCountryName":"US","clientRequestHTTPHost":"bad.example.com"}}]}`, 11, "good.example.com", 11, "bad.example.com")), &zoneData); err != nil { + t.Fatal(err) + } + addHTTPAdaptiveGroups(&zoneData, zone, "account", snapshot, registry) + addFirewallGroupsWithRules(&zoneData, zone, "account", snapshot, registry, map[string]string{"rule-1": "rule-1"}) + + var colo zoneRespColo + if err := json.Unmarshal([]byte(`{"httpRequestsAdaptiveGroups":[{"count":14,"sum":{"visits":15,"edgeResponseBytes":16},"dimensions":{"coloCode":"AMS","clientRequestHTTPHost":"good.example.com"}},{"count":14,"sum":{"visits":15,"edgeResponseBytes":16},"dimensions":{"coloCode":"AMS","clientRequestHTTPHost":"bad.example.com"}}]}`), &colo); err != nil { + t.Fatal(err) + } + for _, group := range colo.ColoGroups { + addColocationGroup(group, zone, "account", snapshot, registry) + } + + var edges zoneRespEdgeErrorsByPath + if err := json.Unmarshal([]byte(`{"httpRequestsAdaptiveGroups":[{"count":17,"dimensions":{"edgeResponseStatus":500,"clientRequestHTTPHost":"good.example.com","clientRequestPath":"/users/1"}},{"count":17,"dimensions":{"edgeResponseStatus":500,"clientRequestHTTPHost":"bad.example.com","clientRequestPath":"/users/2"}}]}`), &edges); err != nil { + t.Fatal(err) + } + addEdgeErrorsByPath(&edges, zone, "account", snapshot, registry) +} + +func resetTask3Metrics(zone string) { + label := prometheus.Labels{"zone": zone} + zoneRequestOriginStatusCountryHost.DeletePartialMatch(label) + zoneRequestOriginStatusCountryHostP50.DeletePartialMatch(label) + zoneRequestOriginStatusCountryHostP95.DeletePartialMatch(label) + zoneRequestOriginStatusCountryHostP99.DeletePartialMatch(label) + zoneRequestStatusCountryHost.DeletePartialMatch(label) + zoneColocationVisits.DeletePartialMatch(label) + zoneColocationEdgeResponseBytes.DeletePartialMatch(label) + zoneColocationRequestsTotal.DeletePartialMatch(label) + zoneFirewallEventsCount.DeletePartialMatch(label) + zoneEdgeErrorsByPath.DeletePartialMatch(label) +} + +func assertTask3Family(t *testing.T, collector prometheus.Collector, labels []string, hosts []string, tuple func(string) []string) { + t.Helper() + all := gatheredHostTuples(t, collector, labels) + zone := tuple(hosts[0])[0] + got := make(map[string]struct{}) + for key := range all { + var values []string + if err := json.Unmarshal([]byte(key), &values); err != nil { + t.Fatal(err) + } + if len(values) > 0 && values[0] == zone { + got[key] = struct{}{} + } + } + want := make(map[string]struct{}, len(hosts)) + for _, host := range hosts { + want[string(mustJSON(tuple(host)))] = struct{}{} + } + if len(got) != len(want) { + t.Fatalf("gathered tuples = %#v, want %d tuples: %#v", got, len(want), want) + } + for key := range want { + if _, ok := got[key]; !ok { + t.Errorf("gathered tuples = %#v, missing %s", got, key) + } + } +} + +func TestAllRegisteredHostMetricFamilies(t *testing.T) { + for _, tt := range []struct { + name string + zone string + snapshot hostWhitelist + want int + }{ + {name: "empty whitelist disabled", zone: "task3-disabled", snapshot: emptyHostWhitelist(), want: 2}, + {name: "empty whitelist enabled", zone: "task3-empty", snapshot: hostWhitelist{enabled: true, allowed: map[string]struct{}{}}, want: 2}, + {name: "one host", zone: "task3-one", snapshot: hostWhitelist{enabled: true, allowed: map[string]struct{}{"good.example.com": {}}}, want: 1}, + {name: "both hosts", zone: "task3-both", snapshot: hostWhitelist{enabled: true, allowed: map[string]struct{}{"good.example.com": {}, "bad.example.com": {}}}, want: 2}, + } { + t.Run(tt.name, func(t *testing.T) { + for _, zone := range []string{"task3-disabled", "task3-empty", "task3-one", "task3-both"} { + resetTask3Metrics(zone) + } + registry := task3Registry() + populateTask3Families(t, tt.zone, tt.snapshot, registry) + registry.Reconcile() + wantHosts := []string{"good.example.com"} + if tt.want == 2 { + wantHosts = append(wantHosts, "bad.example.com") + } + assertTask3Family(t, zoneRequestOriginStatusCountryHost, []string{"zone", "account", "status", "country", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "200", "US", host} }) + assertTask3Family(t, zoneRequestOriginStatusCountryHostP50, []string{"zone", "account", "status", "country", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "200", "US", host} }) + assertTask3Family(t, zoneRequestOriginStatusCountryHostP95, []string{"zone", "account", "status", "country", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "200", "US", host} }) + assertTask3Family(t, zoneRequestOriginStatusCountryHostP99, []string{"zone", "account", "status", "country", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "200", "US", host} }) + assertTask3Family(t, zoneRequestStatusCountryHost, []string{"zone", "account", "status", "country", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "200", "US", host} }) + assertTask3Family(t, zoneColocationVisits, []string{"zone", "account", "colocation", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "AMS", host} }) + assertTask3Family(t, zoneColocationEdgeResponseBytes, []string{"zone", "account", "colocation", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "AMS", host} }) + assertTask3Family(t, zoneColocationRequestsTotal, []string{"zone", "account", "colocation", "host"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "AMS", host} }) + assertTask3Family(t, zoneFirewallEventsCount, []string{"zone", "account", "action", "source", "rule", "host", "country"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "block", "waf", "rule-1", host, "US"} }) + assertTask3Family(t, zoneEdgeErrorsByPath, []string{"zone", "account", "status", "host", "path"}, wantHosts, func(host string) []string { return []string{tt.zone, "account", "500", host, "/users/:id"} }) + + if tt.want == 1 { + labels := prometheus.Labels{"zone": tt.zone, "account": "account", "status": "200", "country": "US", "host": "good.example.com"} + for _, metric := range []struct { + name string + got float64 + want float64 + }{ + {zoneRequestOriginStatusCountryHostMetricName.String(), testutil.ToFloat64(zoneRequestOriginStatusCountryHost.With(labels)), 11}, + {zoneRequestOriginStatusCountryHostP50MetricName.String(), testutil.ToFloat64(zoneRequestOriginStatusCountryHostP50.With(labels)), 101}, + {zoneRequestOriginStatusCountryHostP95MetricName.String(), testutil.ToFloat64(zoneRequestOriginStatusCountryHostP95.With(labels)), 202}, + {zoneRequestOriginStatusCountryHostP99MetricName.String(), testutil.ToFloat64(zoneRequestOriginStatusCountryHostP99.With(labels)), 303}, + {zoneRequestStatusCountryHostMetricName.String(), testutil.ToFloat64(zoneRequestStatusCountryHost.With(labels)), 12}, + } { + if metric.got != metric.want { + t.Errorf("%s = %v, want %v", metric.name, metric.got, metric.want) + } + } + coloLabels := prometheus.Labels{"zone": tt.zone, "account": "account", "colocation": "AMS", "host": "good.example.com"} + for _, metric := range []struct { + name string + got float64 + want float64 + }{ + {zoneColocationRequestsTotalMetricName.String(), testutil.ToFloat64(zoneColocationRequestsTotal.With(coloLabels)), 14}, + {zoneColocationVisitsMetricName.String(), testutil.ToFloat64(zoneColocationVisits.With(coloLabels)), 15}, + {zoneColocationEdgeResponseBytesMetricName.String(), testutil.ToFloat64(zoneColocationEdgeResponseBytes.With(coloLabels)), 16}, + } { + if metric.got != metric.want { + t.Errorf("%s = %v, want %v", metric.name, metric.got, metric.want) + } + } + firewallLabels := prometheus.Labels{"zone": tt.zone, "account": "account", "action": "block", "source": "waf", "rule": "rule-1", "host": "good.example.com", "country": "US"} + if got := testutil.ToFloat64(zoneFirewallEventsCount.With(firewallLabels)); got != 13 { + t.Errorf("%s = %v, want 13", zoneFirewallEventsCountMetricName, got) + } + if got := testutil.ToFloat64(zoneEdgeErrorsByPath.With(prometheus.Labels{"zone": tt.zone, "account": "account", "status": "500", "host": "good.example.com", "path": "/users/:id"})); got != 17 { + t.Errorf("%s = %v, want 17", zoneEdgeErrorsByPathMetricName, got) + } + } + }) + } +} + +func mustJSON(value any) []byte { + b, err := json.Marshal(value) + if err != nil { + panic(err) + } + return b +} diff --git a/main.go b/main.go index 49d1e05d..9c77fa2e 100644 --- a/main.go +++ b/main.go @@ -20,10 +20,13 @@ import ( ) var ( - cfclient *cf.Client - cftimeout time.Duration - gql *GraphQL - log = logrus.New() + cfclient *cf.Client + cftimeout time.Duration + gql *GraphQL + log = logrus.New() + hostWhitelistSnapshot = emptyHostWhitelist() + hostWhitelistMu sync.Mutex + scrapeMu sync.Mutex ) // var ( @@ -112,29 +115,30 @@ func filterExcludedZones(all []cfzones.Zone, exclude []string) []cfzones.Zone { return filtered } -func fetchMetrics() { - var wg sync.WaitGroup +type metricsCollectorRunner func(snapshot hostWhitelist, wg *sync.WaitGroup) + +func runMetricsCollectors(snapshot hostWhitelist, wg *sync.WaitGroup) { targetAccounts := getTargetAccounts() accounts := fetchAccounts(targetAccounts) for _, a := range accounts { wg.Add(1) - go fetchWorkerAnalytics(a, &wg) + go fetchWorkerAnalytics(a, wg) wg.Add(1) - go fetchLogpushAnalyticsForAccount(a, &wg) + go fetchLogpushAnalyticsForAccount(a, wg) wg.Add(1) - go fetchR2StorageForAccount(a, &wg) + go fetchR2StorageForAccount(a, wg) wg.Add(1) - go fetchLoadblancerPoolsHealth(a, &wg) + go fetchLoadblancerPoolsHealth(a, wg) wg.Add(1) - go fetchZeroTrustAnalyticsForAccount(a, &wg) + go fetchZeroTrustAnalyticsForAccount(a, wg) wg.Add(1) - go fetchAccountHTTPDataTransferAnalytics(a, &wg) + go fetchAccountHTTPDataTransferAnalytics(a, wg) } zones := fetchZones(accounts) @@ -149,22 +153,22 @@ func fetchMetrics() { zoneCount := len(filteredZones) if zoneCount > 0 && zoneCount <= cfgraphqlreqlimit { wg.Add(1) - go fetchZoneAnalytics(filteredZones, &wg) + go fetchZoneAnalytics(filteredZones, snapshot, hostSeriesRegistryState, wg) wg.Add(1) - go fetchZoneColocationAnalytics(filteredZones, &wg) + go fetchZoneColocationAnalytics(filteredZones, snapshot, hostSeriesRegistryState, wg) wg.Add(1) - go fetchLoadBalancerAnalytics(filteredZones, &wg) + go fetchLoadBalancerAnalytics(filteredZones, wg) wg.Add(1) - go fetchLogpushAnalyticsForZone(filteredZones, &wg) + go fetchLogpushAnalyticsForZone(filteredZones, wg) wg.Add(1) - go fetchZoneASNAnalytics(filteredZones, &wg) + go fetchZoneASNAnalytics(filteredZones, wg) wg.Add(1) - go fetchEdgeErrorsByPathAnalytics(filteredZones, &wg) + go fetchEdgeErrorsByPathAnalytics(filteredZones, snapshot, hostSeriesRegistryState, wg) } else if zoneCount > cfgraphqlreqlimit { for s := 0; s < zoneCount; s += cfgraphqlreqlimit { e := s + cfgraphqlreqlimit @@ -172,26 +176,43 @@ func fetchMetrics() { e = zoneCount } wg.Add(1) - go fetchZoneAnalytics(filteredZones[s:e], &wg) + go fetchZoneAnalytics(filteredZones[s:e], snapshot, hostSeriesRegistryState, wg) wg.Add(1) - go fetchZoneColocationAnalytics(filteredZones[s:e], &wg) + go fetchZoneColocationAnalytics(filteredZones[s:e], snapshot, hostSeriesRegistryState, wg) wg.Add(1) - go fetchLoadBalancerAnalytics(filteredZones[s:e], &wg) + go fetchLoadBalancerAnalytics(filteredZones[s:e], wg) wg.Add(1) - go fetchLogpushAnalyticsForZone(filteredZones[s:e], &wg) + go fetchLogpushAnalyticsForZone(filteredZones[s:e], wg) wg.Add(1) - go fetchZoneASNAnalytics(filteredZones[s:e], &wg) + go fetchZoneASNAnalytics(filteredZones[s:e], wg) wg.Add(1) - go fetchEdgeErrorsByPathAnalytics(filteredZones[s:e], &wg) + go fetchEdgeErrorsByPathAnalytics(filteredZones[s:e], snapshot, hostSeriesRegistryState, wg) } } +} + +func fetchMetricsWithRunner(run metricsCollectorRunner) { + scrapeMu.Lock() + defer scrapeMu.Unlock() + hostWhitelistMu.Lock() + hostWhitelistSnapshot = loadHostWhitelist(viper.GetString("host_whitelist_path"), hostWhitelistSnapshot) + snapshot := hostWhitelistSnapshot + hostWhitelistMu.Unlock() + hostSeriesRegistryState.ResetScrape() + var wg sync.WaitGroup + run(snapshot, &wg) wg.Wait() + hostSeriesRegistryState.Reconcile() +} + +func fetchMetrics() { + fetchMetricsWithRunner(runMetricsCollectors) } func runExporter() { @@ -221,8 +242,11 @@ func runExporter() { log.Info("Scrape interval set to ", scrapeInterval) go func() { - for ; true; <-time.NewTicker(scrapeInterval).C { - go fetchMetrics() + fetchMetrics() + ticker := time.NewTicker(scrapeInterval) + defer ticker.Stop() + for range ticker.C { + fetchMetrics() } }() @@ -266,6 +290,7 @@ func main() { flags.String("metrics_path", "/metrics", "path for metrics, default /metrics") viper.BindEnv("metrics_path") viper.SetDefault("metrics_path", "/metrics") + viper.SetDefault("host_whitelist_path", hostWhitelistPath) flags.String("cf_api_key", "", "cloudflare api key, required with api_email flag") viper.BindEnv("cf_api_key") diff --git a/prometheus.go b/prometheus.go index c06b3dc5..bbc00983 100644 --- a/prometheus.go +++ b/prometheus.go @@ -378,6 +378,19 @@ var ( }, []string{"account"}) ) +func init() { + hostSeriesRegistryState.Register(zoneRequestOriginStatusCountryHost, "zone", "account", "status", "country", "host") + hostSeriesRegistryState.Register(zoneRequestOriginStatusCountryHostP50, "zone", "account", "status", "country", "host") + hostSeriesRegistryState.Register(zoneRequestOriginStatusCountryHostP95, "zone", "account", "status", "country", "host") + hostSeriesRegistryState.Register(zoneRequestOriginStatusCountryHostP99, "zone", "account", "status", "country", "host") + hostSeriesRegistryState.Register(zoneRequestStatusCountryHost, "zone", "account", "status", "country", "host") + hostSeriesRegistryState.Register(zoneColocationVisits, "zone", "account", "colocation", "host") + hostSeriesRegistryState.Register(zoneColocationEdgeResponseBytes, "zone", "account", "colocation", "host") + hostSeriesRegistryState.Register(zoneColocationRequestsTotal, "zone", "account", "colocation", "host") + hostSeriesRegistryState.Register(zoneFirewallEventsCount, "zone", "account", "action", "source", "rule", "host", "country") + hostSeriesRegistryState.Register(zoneEdgeErrorsByPath, "zone", "account", "status", "host", "path") +} + func buildAllMetricsSet() MetricsSet { allMetricsSet := MetricsSet{} allMetricsSet.Add(zoneRequestTotalMetricName) @@ -733,7 +746,7 @@ func fetchLogpushAnalyticsForZone(zones []cfzones.Zone, wg *sync.WaitGroup) { } } -func fetchZoneColocationAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { +func fetchZoneColocationAnalytics(zones []cfzones.Zone, snapshot hostWhitelist, registry *hostSeriesRegistry, wg *sync.WaitGroup) { defer wg.Done() // Colocation metrics are not available in non-enterprise zones @@ -755,14 +768,39 @@ func fetchZoneColocationAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { cg := z.ColoGroups name, account := findZoneAccountName(zones, z.ZoneTag) for _, c := range cg { - zoneColocationVisits.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.Visits)) - zoneColocationEdgeResponseBytes.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.EdgeResponseBytes)) - zoneColocationRequestsTotal.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Count)) + addColocationGroup(c, name, account, snapshot, registry) } } } -func fetchZoneAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { +func addColocationGroup(c struct { + Dimensions struct { + Datetime string `json:"datetime"` + ColoCode string `json:"coloCode"` + Host string `json:"clientRequestHTTPHost"` + } `json:"dimensions"` + Count uint64 `json:"count"` + Sum struct { + EdgeResponseBytes uint64 `json:"edgeResponseBytes"` + Visits uint64 `json:"visits"` + } `json:"sum"` + Avg struct { + SampleInterval float64 `json:"sampleInterval"` + } `json:"avg"` +}, name string, account string, snapshot hostWhitelist, registry *hostSeriesRegistry) { + labels := prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host} + if !snapshot.Allows(labels["host"]) { + return + } + zoneColocationVisits.With(labels).Add(float64(c.Sum.Visits)) + registry.Observe(zoneColocationVisits, labels) + zoneColocationEdgeResponseBytes.With(labels).Add(float64(c.Sum.EdgeResponseBytes)) + registry.Observe(zoneColocationEdgeResponseBytes, labels) + zoneColocationRequestsTotal.With(labels).Add(float64(c.Count)) + registry.Observe(zoneColocationRequestsTotal, labels) +} + +func fetchZoneAnalytics(zones []cfzones.Zone, snapshot hostWhitelist, registry *hostSeriesRegistry, wg *sync.WaitGroup) { defer wg.Done() // None of the below referenced metrics are available in the free tier @@ -786,9 +824,9 @@ func fetchZoneAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { z := z addHTTPGroups(&z, name, account) - addFirewallGroups(&z, name, account) + addFirewallGroups(&z, name, account, snapshot, registry) addHealthCheckGroups(&z, name, account) - addHTTPAdaptiveGroups(&z, name, account) + addHTTPAdaptiveGroups(&z, name, account, snapshot, registry) } } @@ -861,7 +899,7 @@ func addHTTPGroups(z *zoneResp, name string, account string) { zoneUniquesTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Unique.Uniques)) } -func addFirewallGroups(z *zoneResp, name string, account string) { +func addFirewallGroups(z *zoneResp, name string, account string, snapshot hostWhitelist, registry *hostSeriesRegistry) { // Nothing to do. if len(z.FirewallEventsAdaptiveGroups) == 0 { return @@ -872,17 +910,26 @@ func addFirewallGroups(z *zoneResp, name string, account string) { zoneFirewallEventsCount.DeletePartialMatch(label) rulesMap := fetchFirewallRules(z.ZoneTag) + addFirewallGroupsWithRules(z, name, account, snapshot, registry, rulesMap) +} + +func addFirewallGroupsWithRules(z *zoneResp, name string, account string, snapshot hostWhitelist, registry *hostSeriesRegistry, rulesMap map[string]string) { for _, g := range z.FirewallEventsAdaptiveGroups { + if !snapshot.Allows(g.Dimensions.ClientRequestHTTPHost) { + continue + } + labels := prometheus.Labels{ + "zone": name, + "account": account, + "action": g.Dimensions.Action, + "source": g.Dimensions.Source, + "rule": normalizeRuleName(rulesMap[g.Dimensions.RuleID]), + "host": g.Dimensions.ClientRequestHTTPHost, + "country": g.Dimensions.ClientCountryName, + } zoneFirewallEventsCount.With( - prometheus.Labels{ - "zone": name, - "account": account, - "action": g.Dimensions.Action, - "source": g.Dimensions.Source, - "rule": normalizeRuleName(rulesMap[g.Dimensions.RuleID]), - "host": g.Dimensions.ClientRequestHTTPHost, - "country": g.Dimensions.ClientCountryName, - }).Add(float64(g.Count)) + labels).Add(float64(g.Count)) + registry.Observe(zoneFirewallEventsCount, labels) } } @@ -917,66 +964,50 @@ func addHealthCheckGroups(z *zoneResp, name string, account string) { } } -func addHTTPAdaptiveGroups(z *zoneResp, name string, account string) { +func addHTTPAdaptiveGroups(z *zoneResp, name string, account string, snapshot hostWhitelist, registry *hostSeriesRegistry) { // Clear stale series for this zone/account - label := prometheus.Labels{"zone": name, "account": account} - zoneRequestOriginStatusCountryHost.DeletePartialMatch(label) - zoneRequestOriginStatusCountryHostP50.DeletePartialMatch(label) - zoneRequestOriginStatusCountryHostP95.DeletePartialMatch(label) - zoneRequestOriginStatusCountryHostP99.DeletePartialMatch(label) - zoneRequestStatusCountryHost.DeletePartialMatch(label) for _, g := range z.HTTPRequestsAdaptiveGroups { - zoneRequestOriginStatusCountryHost.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), - "country": g.Dimensions.ClientCountryName, - "host": g.Dimensions.ClientRequestHTTPHost, - }).Add(float64(g.Count)) + labels := prometheus.Labels{ + "zone": name, + "account": account, + "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), + "country": g.Dimensions.ClientCountryName, + "host": g.Dimensions.ClientRequestHTTPHost, + } + if !snapshot.Allows(labels["host"]) { + continue + } + zoneRequestOriginStatusCountryHost.With(labels).Add(float64(g.Count)) + registry.Observe(zoneRequestOriginStatusCountryHost, labels) - zoneRequestOriginStatusCountryHostP50.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), - "country": g.Dimensions.ClientCountryName, - "host": g.Dimensions.ClientRequestHTTPHost, - }).Set(float64(g.Quantile.OriginResponseDurationMsP50)) - - zoneRequestOriginStatusCountryHostP95.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), - "country": g.Dimensions.ClientCountryName, - "host": g.Dimensions.ClientRequestHTTPHost, - }).Set(float64(g.Quantile.OriginResponseDurationMsP95)) - - zoneRequestOriginStatusCountryHostP99.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), - "country": g.Dimensions.ClientCountryName, - "host": g.Dimensions.ClientRequestHTTPHost, - }).Set(float64(g.Quantile.OriginResponseDurationMsP99)) + zoneRequestOriginStatusCountryHostP50.With(labels).Set(float64(g.Quantile.OriginResponseDurationMsP50)) + registry.Observe(zoneRequestOriginStatusCountryHostP50, labels) + + zoneRequestOriginStatusCountryHostP95.With(labels).Set(float64(g.Quantile.OriginResponseDurationMsP95)) + registry.Observe(zoneRequestOriginStatusCountryHostP95, labels) + + zoneRequestOriginStatusCountryHostP99.With(labels).Set(float64(g.Quantile.OriginResponseDurationMsP99)) + registry.Observe(zoneRequestOriginStatusCountryHostP99, labels) } for _, g := range z.HTTPRequestsEdgeCountryHost { - zoneRequestStatusCountryHost.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)), - "country": g.Dimensions.ClientCountryName, - "host": g.Dimensions.ClientRequestHTTPHost, - }).Add(float64(g.Count)) + labels := prometheus.Labels{ + "zone": name, + "account": account, + "status": strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)), + "country": g.Dimensions.ClientCountryName, + "host": g.Dimensions.ClientRequestHTTPHost, + } + if !snapshot.Allows(labels["host"]) { + continue + } + zoneRequestStatusCountryHost.With(labels).Add(float64(g.Count)) + registry.Observe(zoneRequestStatusCountryHost, labels) } } -func fetchEdgeErrorsByPathAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { +func fetchEdgeErrorsByPathAnalytics(zones []cfzones.Zone, snapshot hostWhitelist, registry *hostSeriesRegistry, wg *sync.WaitGroup) { defer wg.Done() if !viper.GetBool("enable_edge_errors_by_path") { @@ -1000,11 +1031,11 @@ func fetchEdgeErrorsByPathAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup) { for _, z := range r.Viewer.Zones { name, account := findZoneAccountName(zones, z.ZoneTag) - addEdgeErrorsByPath(&z, name, account) + addEdgeErrorsByPath(&z, name, account, snapshot, registry) } } -func addEdgeErrorsByPath(z *zoneRespEdgeErrorsByPath, name string, account string) { +func addEdgeErrorsByPath(z *zoneRespEdgeErrorsByPath, name string, account string, snapshot hostWhitelist, registry *hostSeriesRegistry) { if len(z.HTTPRequestsAdaptiveGroups) == 0 { return } @@ -1013,14 +1044,18 @@ func addEdgeErrorsByPath(z *zoneRespEdgeErrorsByPath, name string, account strin zoneEdgeErrorsByPath.DeletePartialMatch(label) for _, g := range z.HTTPRequestsAdaptiveGroups { - zoneEdgeErrorsByPath.With( - prometheus.Labels{ - "zone": name, - "account": account, - "status": strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)), - "host": g.Dimensions.ClientRequestHTTPHost, - "path": normalizePath(g.Dimensions.ClientRequestPath), - }).Add(float64(g.Count)) + labels := prometheus.Labels{ + "zone": name, + "account": account, + "status": strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)), + "host": g.Dimensions.ClientRequestHTTPHost, + "path": normalizePath(g.Dimensions.ClientRequestPath), + } + if !snapshot.Allows(labels["host"]) { + continue + } + zoneEdgeErrorsByPath.With(labels).Add(float64(g.Count)) + registry.Observe(zoneEdgeErrorsByPath, labels) } }