diff --git a/api/cmd/main.go b/api/cmd/main.go index 0759c8576..c199bc30a 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -354,9 +354,10 @@ func main() { // What the telemetry reporter and the admin preview both collect from. telemetryDeps := telemetry.Deps{ - Kube: k8s, - Store: store, - Version: Version, + Kube: k8s, + Registry: reg, + Store: store, + Version: Version, Flags: telemetry.Flags{ CaptureEnabled: cfg.captureFeatureEnabled, OIDCConfigured: cfg.oidcIssuer != "", diff --git a/api/internal/handlers/telemetry.go b/api/internal/handlers/telemetry.go index 9099d4b4f..eec662c38 100644 --- a/api/internal/handlers/telemetry.go +++ b/api/internal/handlers/telemetry.go @@ -31,7 +31,7 @@ const ( // first-login notice lists (contracts/report-schema.md). var ( noticeBasicFields = []string{"version", "servers", "templates"} - noticeExtendedFields = []string{"installId", "env.k8s", "env.distro", "env.arch", "env.nodes", "games", "features"} + noticeExtendedFields = []string{"installId", "env.k8s", "env.distro", "env.arch", "env.nodes", "env.deployment", "games", "features"} ) // TelemetrySettings is the install-time telemetry setting the config and diff --git a/api/internal/handlers/telemetry_envtest_test.go b/api/internal/handlers/telemetry_envtest_test.go index 504756bd1..22086ae25 100644 --- a/api/internal/handlers/telemetry_envtest_test.go +++ b/api/internal/handlers/telemetry_envtest_test.go @@ -127,8 +127,8 @@ func TestTelemetryNotice_PendingOnlyForManagersOnADefaultInstallWithADestination fields, _ := got["fields"].(map[string]any) basic, _ := fields["basic"].([]any) extended, _ := fields["extended"].([]any) - if len(basic) != 3 || len(extended) != 7 { - t.Fatalf("fields = %v, want 3 basic and 7 extended", fields) + if len(basic) != 3 || len(extended) != 8 { + t.Fatalf("fields = %v, want 3 basic and 8 extended", fields) } }) diff --git a/api/internal/telemetry/collect.go b/api/internal/telemetry/collect.go index 1d0a55007..f6f13aa93 100644 --- a/api/internal/telemetry/collect.go +++ b/api/internal/telemetry/collect.go @@ -7,11 +7,14 @@ import ( "fmt" "regexp" "sort" + "sync" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" "github.com/GameplanePanel/gameplane/api/internal/db" "github.com/GameplanePanel/gameplane/api/internal/kube" @@ -19,11 +22,21 @@ import ( ) // extSchema is the extended-part schema version this build emits. -const extSchema = 1 +const extSchema = 2 // other is the value every enumerated field falls back to. const other = "other" +// clusterTimeout bounds the calls the report makes to one managed cluster, so +// a cluster that doesn't answer can't hold the report past its slot (FR-044). +const clusterTimeout = 10 * time.Second + +// Deployment markers (FR-045). +const ( + deploymentCluster = "cluster" + deploymentStandalone = "standalone" +) + // nonDigits matches a run of non-digits, which is stripped from the // discovery minor version ("31+" on EKS and GKE becomes "31"). var nonDigits = regexp.MustCompile(`[^0-9]+`) @@ -49,35 +62,53 @@ type Flags struct { // Deps is what Collect reads from. Kube and Store are required. type Deps struct { - Kube *kube.Client - Store *db.Store - Flags Flags - Version string + Kube *kube.Client + // Registry holds every cluster the panel manages: the local cluster, when + // there is one, and each registered remote cluster. Nil means the local + // cluster (Kube) only. + Registry *kube.Registry + Store *db.Store + Flags Flags + Version string // Extended asks for the extended part (the gate's extendedOn). Extended bool // Now returns the current time; nil means time.Now. Tests set it. Now func() time.Time } +// clusterData is what one reachable managed cluster contributes to a report. +// The fields after templates are read only for an extended report. +type clusterData struct { + servers []unstructured.Unstructured + templates []unstructured.Unstructured + nodes []corev1.Node + nodesOK bool + gitVersion string + minor string + games telemetryschema.Games + backups bool +} + // Collect builds the report the reporter would send now, and the same value -// backs the admin preview. The basic counts cover the local cluster only. -// When deps.Extended is set it also fills the extended part from the sources -// in research R14. A field whose source can't be read yields 0, false or -// "other" rather than an error; the only errors are a missing install ID and -// a failure to derive the signing key, because the part can't be signed -// without them. +// backs the admin preview. The counts cover every managed cluster that can +// be listed: the local cluster, when there is one, and every registered +// remote cluster (FR-043); a cluster that can't be listed is left out +// (FR-044). When deps.Extended is set it also fills the extended part from +// the sources in research R14 and R23. A field whose source can't be read +// yields 0, false or "other" rather than an error; the only errors are a +// missing install ID and a failure to derive the signing key, because the +// part can't be signed without them. func Collect(ctx context.Context, deps Deps) (telemetryschema.Report, error) { - servers := listItems(ctx, deps, "servers") - templates := listItems(ctx, deps, "templates") - rep := telemetryschema.Report{ - Version: deps.Version, - Servers: len(servers), - Templates: len(templates), + clusters := readClusters(ctx, deps) + rep := telemetryschema.Report{Version: deps.Version} + for _, c := range clusters { + rep.Servers += len(c.servers) + rep.Templates += len(c.templates) } if !deps.Extended { return rep, nil } - ext, err := collectExtended(ctx, deps, servers, templates) + ext, err := collectExtended(ctx, deps, clusters) if err != nil { return telemetryschema.Report{}, err } @@ -85,23 +116,107 @@ func Collect(ctx context.Context, deps Deps) (telemetryschema.Report, error) { return rep, nil } -// listItems lists every object of one kube.GVRs kind across namespaces. A -// failure yields no items, so the count is 0. -func listItems(ctx context.Context, deps Deps, kind string) []unstructured.Unstructured { +// managedClusters returns the clients of the clusters the report covers: +// every cluster in deps.Registry, or the local cluster when there is no +// registry. A standalone panel's management client has no Kubernetes +// transport and is never one of them. +func managedClusters(deps Deps) []*kube.Client { + if deps.Registry == nil { + if deps.Kube == nil || deps.Kube.IsStandalone() { + return nil + } + return []*kube.Client{deps.Kube} + } + var out []*kube.Client + for _, id := range deps.Registry.IDs() { + if c, ok := deps.Registry.Get(id); ok && c != nil && !c.IsStandalone() { + out = append(out, c) + } + } + return out +} + +// readClusters reads every managed cluster at once, each under its own +// clusterTimeout, so slow clusters can't add up (FR-044). It lists GameServers +// and GameTemplates and, for an extended report, nodes, the server version, +// game counts and backup schedules. A cluster whose GameServer, GameTemplate +// or (for an extended report) backup schedule list fails is left out of the +// report (R23), so its servers are never counted with partial data. The result +// keeps the registry's order. +func readClusters(ctx context.Context, deps Deps) []clusterData { + clients := managedClusters(deps) + results := make([]*clusterData, len(clients)) + var wg sync.WaitGroup + for i, c := range clients { + wg.Add(1) + go func() { + defer wg.Done() + cd := deps + cd.Kube = c + cctx, cancel := context.WithTimeout(ctx, clusterTimeout) + defer cancel() + servers, err := listKind(cctx, cd, "servers") + if err != nil { + return + } + templates, err := listKind(cctx, cd, "templates") + if err != nil { + return + } + r := &clusterData{servers: servers, templates: templates} + if deps.Extended { + r.nodes, r.nodesOK = listNodes(cctx, cd) + r.gitVersion, r.minor = serverVersionCtx(cctx, cd) + r.games = countGames(cctx, cd, r.servers, r.templates) + schedules, err := listKind(cctx, cd, "schedules") + if err != nil { + return + } + r.backups = len(schedules) > 0 + } + results[i] = r + }() + } + wg.Wait() + out := make([]clusterData, 0, len(results)) + for _, r := range results { + if r != nil { + out = append(out, *r) + } + } + return out +} + +// listKind lists every object of one kube.GVRs kind across namespaces on +// deps.Kube. It fails when the kind is unknown, there is no dynamic client, +// or the list call fails. +func listKind(ctx context.Context, deps Deps, kind string) ([]unstructured.Unstructured, error) { gvr, ok := kube.GVRs[kind] - if !ok || deps.Kube == nil || deps.Kube.Dynamic == nil { - return nil + if !ok { + return nil, fmt.Errorf("list %s: unknown kind", kind) + } + if deps.Kube == nil || deps.Kube.Dynamic == nil { + return nil, fmt.Errorf("list %s: no cluster client", kind) } list, err := deps.Kube.Dynamic.Resource(gvr).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("list %s: %w", kind, err) + } + return list.Items, nil +} + +// listItems is listKind with a failure read as no items, so the count is 0. +func listItems(ctx context.Context, deps Deps, kind string) []unstructured.Unstructured { + items, err := listKind(ctx, deps, kind) if err != nil { return nil } - return list.Items + return items } -// collectExtended builds the extended part and signs it with the key derived -// from the install ID and the stored secret. -func collectExtended(ctx context.Context, deps Deps, servers, templates []unstructured.Unstructured) (*telemetryschema.Extended, error) { +// collectExtended builds the extended part over the reachable clusters and +// signs it with the key derived from the install ID and the stored secret. +func collectExtended(ctx context.Context, deps Deps, clusters []clusterData) (*telemetryschema.Extended, error) { now := time.Now if deps.Now != nil { now = deps.Now @@ -122,30 +237,70 @@ func collectExtended(ctx context.Context, deps Deps, servers, templates []unstru return nil, fmt.Errorf("collect telemetry: derive key: %w", err) } - nodes, nodesOK := listNodes(ctx, deps) - gitVersion, minor := serverVersion(deps) + // Environment over every reachable cluster (OD-15, OD-16): k8s and + // distro are sets, arch is the union, and the node band counts all nodes. + k8s, distros := map[string]bool{}, map[string]bool{} + var allNodes []corev1.Node + var servers []unstructured.Unstructured + nodesOK, backups := false, false + games := telemetryschema.Games{Official: map[string]int{}} + for _, c := range clusters { + if c.nodesOK { + nodesOK = true + allNodes = append(allNodes, c.nodes...) + } + backups = backups || c.backups + k8s[c.minor] = true + distros[detectDistro(c.gitVersion, c.nodes)] = true + for m, n := range c.games.Official { + games.Official[m] += n + } + games.Custom += c.games.Custom + servers = append(servers, c.servers...) + } nodeBand := other - if nodesOK && len(nodes) > 0 { - nodeBand = telemetryschema.NodeBand(len(nodes)) + if nodesOK && len(allNodes) > 0 { + nodeBand = telemetryschema.NodeBand(len(allNodes)) } + features := collectFeatures(ctx, deps, servers) + features.Backups = backups return &telemetryschema.Extended{ Schema: extSchema, InstallID: state.InstallID, Env: telemetryschema.Env{ - K8s: minor, - Distro: detectDistro(gitVersion, nodes), - Arch: nodeArches(nodes), - Nodes: nodeBand, + K8s: setOrOther(k8s), + Distro: setOrOther(distros), + Arch: nodeArches(allNodes), + Nodes: nodeBand, + Deployment: deployment(deps), }, - Games: countGames(ctx, deps, servers, templates), - Features: collectFeatures(ctx, deps, servers), + Games: games, + Features: features, Key: telemetryschema.PublicKeyString(priv), SentAt: now().UTC().Truncate(time.Second), }, nil } -// listNodes lists the local cluster's nodes. ok is false when the call failed. +// deployment is the FR-045 marker: standalone when the panel runs without a +// local cluster, cluster otherwise. +func deployment(deps Deps) string { + if deps.Kube.IsStandalone() { + return deploymentStandalone + } + return deploymentCluster +} + +// setOrOther returns the sorted members of set, or ["other"] when it is +// empty, because the wire schema rejects an empty list. +func setOrOther(set map[string]bool) []string { + if len(set) == 0 { + return []string{other} + } + return sortedKeys(set) +} + +// listNodes lists the nodes of deps.Kube. ok is false when the call failed. func listNodes(ctx context.Context, deps Deps) ([]corev1.Node, bool) { if deps.Kube == nil || deps.Kube.Typed == nil { return nil, false @@ -157,6 +312,24 @@ func listNodes(ctx context.Context, deps Deps) ([]corev1.Node, bool) { return list.Items, true } +// serverVersionCtx is serverVersion bounded by ctx: the discovery call takes +// no context, so it runs aside and an expired ctx reads as "other". The call +// itself is bounded by discoveryClient's timeout. +func serverVersionCtx(ctx context.Context, deps Deps) (gitVersion, minor string) { + type result struct{ git, minor string } + ch := make(chan result, 1) + go func() { + g, m := serverVersion(deps) + ch <- result{g, m} + }() + select { + case r := <-ch: + return r.git, r.minor + case <-ctx.Done(): + return "", other + } +} + // serverVersion returns the API server's gitVersion (for distro detection) // and its minor version as "1.N", or "other" when it can't be read or doesn't // fit the 1.N pattern. Non-digit suffixes such as the "+" of "31+" are @@ -165,7 +338,7 @@ func serverVersion(deps Deps) (gitVersion, minor string) { if deps.Kube == nil || deps.Kube.Typed == nil { return "", other } - v, err := deps.Kube.Typed.Discovery().ServerVersion() + v, err := discoveryClient(deps.Kube).ServerVersion() if err != nil || v == nil { return "", other } @@ -176,6 +349,23 @@ func serverVersion(deps Deps) (gitVersion, minor string) { return v.GitVersion, minor } +// discoveryClient returns a discovery client for c whose requests time out +// after clusterTimeout, so a hung cluster can't hold a discovery call open +// after the report gave up on it. Clients built without a rest.Config (test +// fakes) use their own discovery client. +func discoveryClient(c *kube.Client) discovery.DiscoveryInterface { + if c.Config == nil { + return c.Typed.Discovery() + } + cfg := rest.CopyConfig(c.Config) + cfg.Timeout = clusterTimeout + d, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return c.Typed.Discovery() + } + return d +} + // nodeArches returns the sorted, unique CPU architectures of nodes. Values // outside the enumeration become "other", and a cluster with no readable // node reports ["other"] because the wire schema rejects an empty list. @@ -230,8 +420,10 @@ func countGames(ctx context.Context, deps Deps, servers, templates []unstructure return games } -// collectFeatures reads the feature flags and bands from the local cluster -// and the API's own configuration. +// collectFeatures reads the feature flags and bands from servers, the +// GameServers of every reachable cluster, and the API's own configuration. +// Backups is left false; collectExtended sets it over every reachable +// cluster. func collectFeatures(ctx context.Context, deps Deps, servers []unstructured.Unstructured) telemetryschema.Features { tunnels := map[string]bool{} wake := false @@ -248,7 +440,6 @@ func collectFeatures(ctx context.Context, deps Deps, servers []unstructured.Unst WakeOnConnect: wake, Tunnels: sortedKeys(tunnels), Capture: deps.Flags.CaptureEnabled, - Backups: len(listItems(ctx, deps, "schedules")) > 0, SSO: deps.Flags.OIDCConfigured || hasSSOProvider(ctx, deps.Store), AuditForwarding: deps.Flags.AuditWebhook || deps.Flags.AuditS3, Clusters: telemetryschema.ClusterBand(localClusterCount(deps) + countClusters(ctx, deps)), diff --git a/api/internal/telemetry/collect_clusters_test.go b/api/internal/telemetry/collect_clusters_test.go new file mode 100644 index 000000000..84a714237 --- /dev/null +++ b/api/internal/telemetry/collect_clusters_test.go @@ -0,0 +1,158 @@ +package telemetry + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/version" + + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/telemetryschema" +) + +// TestCollect_SumsReachableClustersAndSkipsUnreachable checks that the counts +// and the environment cover every reachable registered cluster, and that a +// cluster whose list fails is left out (SC-018). +func TestCollect_SumsReachableClustersAndSkipsUnreachable(t *testing.T) { + local := collectKubeClients(t, collectOpts{ + nodes: []corev1.Node{testNode("l1", nil, "", "Ubuntu 24.04", "amd64")}, + serverInfo: &version.Info{Major: "1", Minor: "31", GitVersion: "v1.31.2+k3s1"}, + dynamic: []runtime.Object{ + serverObj("ls1", "lt", false, "", false), + templateObj("lt", nil), + }, + }) + remote := collectKubeClients(t, collectOpts{ + nodes: []corev1.Node{ + testNode("r1", nil, "", "Amazon Linux 2023", "arm64"), + testNode("r2", nil, "", "Amazon Linux 2023", "arm64"), + }, + serverInfo: &version.Info{Major: "1", Minor: "30", GitVersion: "v1.30.4-eks-abc"}, + dynamic: []runtime.Object{ + serverObj("rs1", "rt", false, "", false), + serverObj("rs2", "rt", false, "", false), + templateObj("rt", nil), + gpObject("BackupSchedule", "nightly", "gameplane-games"), + }, + }) + dead := collectKubeClients(t, collectOpts{ + nodesFail: true, + versionFail: true, + dynamicFail: map[string]bool{"list": true}, + }) + + reg := kube.NewRegistry("local") + reg.Set("local", local) + reg.Set("remote-a", remote) + reg.Set("remote-b", dead) + + store := collectStore(t, true) + deps := Deps{Kube: local, Registry: reg, Store: store, Version: "v1", Extended: true, Now: fixedNow} + rep, err := Collect(t.Context(), deps) + if err != nil { + t.Fatalf("collect: %v", err) + } + if rep.Servers != 3 || rep.Templates != 2 { + t.Fatalf("counts = %d servers, %d templates; want 3 and 2 (the dead cluster is skipped)", rep.Servers, rep.Templates) + } + if rep.Ext == nil { + t.Fatal("missing extended part") + } + wantEnv := telemetryschema.Env{ + K8s: []string{"1.30", "1.31"}, + Distro: []string{"eks", "k3s"}, + Arch: []string{"amd64", "arm64"}, + Nodes: telemetryschema.NodeBand(3), + Deployment: "cluster", + } + if !reflect.DeepEqual(rep.Ext.Env, wantEnv) { + t.Errorf("env = %+v, want %+v", rep.Ext.Env, wantEnv) + } + if !rep.Ext.Features.Backups { + t.Error("a backup schedule on a reachable cluster must set backups") + } + if rep.Ext.Games.Custom != 3 { + t.Errorf("games.custom = %d, want 3", rep.Ext.Games.Custom) + } + if rep.Ext.Schema != 2 { + t.Errorf("schema = %d, want 2", rep.Ext.Schema) + } + + basic, err := Collect(t.Context(), Deps{Kube: local, Registry: reg, Store: store, Version: "v1", Extended: false, Now: fixedNow}) + if err != nil { + t.Fatalf("basic collect: %v", err) + } + if basic.Servers != 3 || basic.Templates != 2 { + t.Fatalf("basic counts = %d servers, %d templates; want 3 and 2", basic.Servers, basic.Templates) + } + if basic.Ext != nil { + t.Fatal("the extended part must be absent when Extended is false") + } +} + +// TestCollect_StandaloneMarksDeploymentAndCountsRegisteredOnly checks that a +// standalone panel reports deployment "standalone" and counts only the +// registered clusters, never its own management client. +func TestCollect_StandaloneMarksDeploymentAndCountsRegisteredOnly(t *testing.T) { + mgmt := &kube.Client{ClusterStore: collectKubeClients(t, collectOpts{}).Dynamic.Resource(kube.GVRCluster)} + + t.Run("registered remote is counted", func(t *testing.T) { + remote := collectKubeClients(t, collectOpts{ + nodes: []corev1.Node{testNode("r1", nil, "", "Ubuntu 24.04", "amd64")}, + serverInfo: &version.Info{Major: "1", Minor: "31", GitVersion: "v1.31.2+k3s1"}, + dynamic: []runtime.Object{ + serverObj("rs1", "rt", false, "", false), + templateObj("rt", nil), + }, + }) + reg := kube.NewRegistry("remote") + reg.Set("remote", remote) + + rep, err := Collect(t.Context(), Deps{ + Kube: mgmt, Registry: reg, Store: collectStore(t, true), Version: "v1", Extended: true, Now: fixedNow, + }) + if err != nil { + t.Fatalf("collect: %v", err) + } + if rep.Servers != 1 || rep.Templates != 1 { + t.Fatalf("counts = %d servers, %d templates; want 1 and 1", rep.Servers, rep.Templates) + } + if rep.Ext == nil { + t.Fatal("missing extended part") + } + if rep.Ext.Env.Deployment != "standalone" { + t.Errorf("deployment = %q, want standalone", rep.Ext.Env.Deployment) + } + if !reflect.DeepEqual(rep.Ext.Env.K8s, []string{"1.31"}) { + t.Errorf("k8s = %v, want [1.31]", rep.Ext.Env.K8s) + } + }) + + t.Run("empty registry reports fallbacks", func(t *testing.T) { + rep, err := Collect(t.Context(), Deps{ + Kube: mgmt, Registry: kube.NewRegistry("local"), Store: collectStore(t, true), + Version: "v1", Extended: true, Now: fixedNow, + }) + if err != nil { + t.Fatalf("collect: %v", err) + } + if rep.Servers != 0 || rep.Templates != 0 { + t.Fatalf("counts = %d servers, %d templates; want 0 and 0", rep.Servers, rep.Templates) + } + if rep.Ext == nil { + t.Fatal("missing extended part") + } + want := telemetryschema.Env{ + K8s: []string{"other"}, + Distro: []string{"other"}, + Arch: []string{"other"}, + Nodes: "other", + Deployment: "standalone", + } + if !reflect.DeepEqual(rep.Ext.Env, want) { + t.Errorf("env = %+v, want %+v", rep.Ext.Env, want) + } + }) +} diff --git a/api/internal/telemetry/collect_test.go b/api/internal/telemetry/collect_test.go index 95afceeed..b9d1891a3 100644 --- a/api/internal/telemetry/collect_test.go +++ b/api/internal/telemetry/collect_test.go @@ -210,7 +210,7 @@ func TestCollect_NilClients(t *testing.T) { if rep.Servers != 0 || rep.Templates != 0 || rep.Ext == nil { t.Fatalf("%s: report = %+v", name, rep) } - if rep.Ext.Env.K8s != "other" || rep.Ext.Env.Nodes != "other" || !reflect.DeepEqual(rep.Ext.Env.Arch, []string{"other"}) { + if !reflect.DeepEqual(rep.Ext.Env.K8s, []string{"other"}) || rep.Ext.Env.Nodes != "other" || !reflect.DeepEqual(rep.Ext.Env.Arch, []string{"other"}) { t.Errorf("%s: env = %+v, want other fallbacks", name, rep.Ext.Env) } } @@ -272,10 +272,10 @@ func TestCollect_ExtendedFullReport(t *testing.T) { if ext == nil { t.Fatal("missing extended part") } - if ext.Schema != 1 || ext.InstallID != testInstallID { + if ext.Schema != 2 || ext.InstallID != testInstallID { t.Errorf("schema/id = %d/%q", ext.Schema, ext.InstallID) } - wantEnv := telemetryschema.Env{K8s: "1.31", Distro: "k3s", Arch: []string{"amd64", "arm64"}, Nodes: telemetryschema.NodeBand(2)} + wantEnv := telemetryschema.Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64", "arm64"}, Nodes: telemetryschema.NodeBand(2), Deployment: "cluster"} if !reflect.DeepEqual(ext.Env, wantEnv) { t.Errorf("env = %+v, want %+v", ext.Env, wantEnv) } @@ -315,7 +315,7 @@ func TestCollect_ExtendedFieldsFallBackWhenSourcesFail(t *testing.T) { if ext == nil { t.Fatal("missing extended part") } - wantEnv := telemetryschema.Env{K8s: "other", Distro: "other", Arch: []string{"other"}, Nodes: "other"} + wantEnv := telemetryschema.Env{K8s: []string{"other"}, Distro: []string{"other"}, Arch: []string{"other"}, Nodes: "other", Deployment: "cluster"} if !reflect.DeepEqual(ext.Env, wantEnv) { t.Errorf("env = %+v, want %+v", ext.Env, wantEnv) } diff --git a/api/specs.md b/api/specs.md index b5aec0b72..1dccc8987 100644 --- a/api/specs.md +++ b/api/specs.md @@ -18,7 +18,7 @@ The api module is the REST + WebSocket gateway for Gameplane, serving the web da - **Module registries:** pluggable providers (CurseForge, Modrinth, Spigot, Hangar, Nexus, Steam, Thunderstore, Factorio, GitHub, UMod); live lookup + caching - **Multi-cluster dispatch:** `?cluster=` selector routes requests to remote clusters via registered kubeconfigs; home cluster is `local` - **Unified reads:** `/fleet/*` aggregates only the caller's authorized resources across registered clusters, with explicit target identities, target permissions, partial failures and bounded work; it does not schedule or mutate resources. -- **Telemetry:** daily usage reports POSTed to the resolved destination (operator hard-off, bundled receiver, custom endpoint, or the project's default receiver). Two admin-consented tiers: basic (`version`, `servers`, `templates`) and extended (random install ID, environment categories, game and feature counts, signing key, send time; reports are signed). New installs share both tiers by default once the first-login notice has been shown to an admin; nothing is sent before then. The schedule and consent source live in `telemetry_state` (migration 015), so restarts never send early and replicas share one schedule. When the destination offers proof-of-work challenges (`GET /v1/challenge`), the reporter solves one per request (a challenge above 26 bits is refused) +- **Telemetry:** daily usage reports POSTed to the resolved destination (operator hard-off, bundled receiver, custom endpoint, or the project's default receiver). Two admin-consented tiers: basic (`version`, `servers`, `templates`) and extended (random install ID, environment categories, game and feature counts, signing key, send time; reports are signed). Counts cover every managed cluster: each cluster in the registry except a standalone panel's own client, each read with a 10 s timeout; a cluster whose GameServer, GameTemplate or (extended only) backup schedule list fails is left out of that report. Servers, templates and game counts are summed; extended `env.k8s` and `env.distro` are sets over the reachable clusters, `env.arch` their union, `env.nodes` one band over their total node count, and `env.deployment` is `standalone` or `cluster` (extended schema 2). New installs share both tiers by default once the first-login notice has been shown to an admin; nothing is sent before then. The schedule and consent source live in `telemetry_state` (migration 015), so restarts never send early and replicas share one schedule. When the destination offers proof-of-work challenges (`GET /v1/challenge`), the reporter solves one per request (a challenge above 26 bits is refused) ## Non-goals / boundaries diff --git a/docs/install.md b/docs/install.md index 3cd82ebef..b4da5a2a3 100644 --- a/docs/install.md +++ b/docs/install.md @@ -442,15 +442,19 @@ described below, so you can decide before you install. - **Basic** — exactly three fields: `version` (the Gameplane version), `servers` (how many GameServers exist) and `templates` (how many - GameTemplates exist). No names, namespaces, hostnames, or addresses. + GameTemplates exist), summed over the local cluster, when there is one, + and every registered cluster the panel can reach. No names, namespaces, + hostnames, or addresses. - **Extended** — the basic fields plus the following, and nothing else: - a random **install ID** (a UUID generated on your install; it is not derived from your cluster, host, network, or users) - - environment: Kubernetes minor version (for example `1.31`), a - distribution category (`k3s`, `rke2`, `k0s`, `eks`, `gke`, `aks`, - `openshift`, `microk8s`, `minikube`, `kind`, `doks`, `talos` or - `other`), node CPU architectures, and a node-count band (`1`, `2-3`, - `4-10`, `11-50`, `51+`) + - environment, over every cluster the panel can reach: the Kubernetes + minor versions (for example `1.31`), the distribution categories + (`k3s`, `rke2`, `k0s`, `eks`, `gke`, `aks`, `openshift`, `microk8s`, + `minikube`, `kind`, `doks`, `talos` or `other`), node CPU + architectures, a node-count band over all their nodes (`1`, `2-3`, + `4-10`, `11-50`, `51+`), and whether the panel runs on a cluster or + standalone (`cluster` or `standalone`) - game usage: a server count for each module from the official catalog, plus one combined `custom` count for every other module (custom module names are never sent) diff --git a/specs/022-default-telemetry-dashboard/tasks.md b/specs/022-default-telemetry-dashboard/tasks.md index 28d8f4614..3287e4cb9 100644 --- a/specs/022-default-telemetry-dashboard/tasks.md +++ b/specs/022-default-telemetry-dashboard/tasks.md @@ -609,12 +609,12 @@ description: "Task list for feature 022: default telemetry destination, extended **Independent Test**: SC-018. -- [ ] T121 [P] In `telemetryschema/`: raise `SchemaVersion` to `2`, add `Env.Deployment` and the enumeration `Deployments = {"cluster", "standalone"}` in `enums.go`, and make `decodeEnv` require `deployment` on schema 2 and refuse it on schema 1 (unknown key). Schema 1 reports decode with `Deployment` set to `other` (OD-14). In schema 2, `env.k8s` and `env.distro` become non-empty sorted sets like `env.arch` (OD-15), and a schema 1 string decodes as a one-value set; `env.nodes` stays one band (OD-16). Update `telemetryschema/specs.md`. -- [ ] T122 [US5] Design first, on the local machine where Pencil runs: in `telemetry-receiver/telemetry-dashboard.pen`, add a "Deployment" share card to the environment breakdowns of frame `i0xK1E`, matching the other environment cards. Export it into `telemetry-receiver/design-export/` and commit the export with the design (rules 1, 2). -- [ ] T123 [US5] Receiver, after T121: add `dimDeployment` to the `daily_dim` writes in `ingest_ext.go` and write one row per `k8s` and `distro` value as for `arch`, `Env.Deployment` to `views.go` (`extended.env.deployment`), and after T122 the card in `charts_ext.go` and `web/overview_extended.html`. Update `telemetry-receiver/specs.md` and `README.md`. -- [ ] T124 API, after T121: give `telemetry.Deps` the cluster registry (`api/cmd/main.go:352`), and in `api/internal/telemetry/collect.go` list servers, templates and backup schedules on every cluster in `Registry.IDs()` with a per-cluster timeout, sum them, skip clusters whose list fails (FR-043, FR-044), and set `Env.Deployment` from `IsStandalone()` (FR-045). Read the server version and nodes of every reachable cluster: `k8s` and `distro` as sets, `arch` as the union (OD-15), `nodes` as one band over their total node count (OD-16). Bump `extSchema` to `2`. +- [X] T121 [P] In `telemetryschema/`: raise `SchemaVersion` to `2`, add `Env.Deployment` and the enumeration `Deployments = {"cluster", "standalone"}` in `enums.go`, and make `decodeEnv` require `deployment` on schema 2 and refuse it on schema 1 (unknown key). Schema 1 reports decode with `Deployment` set to `other` (OD-14). In schema 2, `env.k8s` and `env.distro` become non-empty sorted sets like `env.arch` (OD-15), and a schema 1 string decodes as a one-value set; `env.nodes` stays one band (OD-16). Update `telemetryschema/specs.md`. +- [X] T122 [US5] Design first, on the local machine where Pencil runs: in `telemetry-receiver/telemetry-dashboard.pen`, add a "Deployment" share card to the environment breakdowns of frame `i0xK1E`, matching the other environment cards. Export it into `telemetry-receiver/design-export/` and commit the export with the design (rules 1, 2). +- [X] T123 [US5] Receiver, after T121: add `dimDeployment` to the `daily_dim` writes in `ingest_ext.go` and write one row per `k8s` and `distro` value as for `arch`, `Env.Deployment` to `views.go` (`extended.env.deployment`), and after T122 the card in `charts_ext.go` and `web/overview_extended.html`. Update `telemetry-receiver/specs.md` and `README.md`. +- [X] T124 API, after T121: give `telemetry.Deps` the cluster registry (`api/cmd/main.go:352`), and in `api/internal/telemetry/collect.go` list servers, templates and backup schedules on every cluster in `Registry.IDs()` with a per-cluster timeout, sum them, skip clusters whose list fails (FR-043, FR-044), and set `Env.Deployment` from `IsStandalone()` (FR-045). Read the server version and nodes of every reachable cluster: `k8s` and `distro` as sets, `arch` as the union (OD-15), `nodes` as one band over their total node count (OD-16). Bump `extSchema` to `2`. - [ ] T125 [P] Add `env.deployment` to `noticeExtendedFields` in `api/internal/handlers/telemetry.go`, to the `env` type in `web/src/lib/api.ts`, and to the extended field list in `docs/` and the website data-handling statement (submodule PR flow, then a pointer bump). -- [ ] T126 Tests (CI only, rule 8): new tests for schema 1 and 2 decoding, the deployment dimension, cross-cluster counting with one unreachable cluster, and the standalone marker (SC-018). Existing tests whose expectations change (for example a schema version or field list they assert) need the user's sign-off before editing, with the exact before and after. +- [X] T126 Tests (CI only, rule 8): new tests for schema 1 and 2 decoding, the deployment dimension, cross-cluster counting with one unreachable cluster, and the standalone marker (SC-018). Existing tests whose expectations change (for example a schema version or field list they assert) need the user's sign-off before editing, with the exact before and after. - [ ] T127 Ship through the open PR or a new one, as the user decides when the phase starts; labels `type: feature`, `area: api`, `area: shared`, `area: specs` (plus `area: web` for T125) through REST (rule 14). --- diff --git a/telemetry-receiver/README.md b/telemetry-receiver/README.md index ae52f235e..a9ac3a199 100644 --- a/telemetry-receiver/README.md +++ b/telemetry-receiver/README.md @@ -83,6 +83,12 @@ version lower than the newest one seen in the range. The trend is stored as daily counts only (`daily_install_version`, `daily_version_change`), never per install. +The environment cards count an install once under each value it reports +across the clusters its panel manages, so Kubernetes version, distribution +and architecture shares can add up to more than 100%. "Deployment" shows +whether the panel runs on a cluster or standalone; installs on an older +version that doesn't report it count as other. + ### Metrics On the dashboard listener, Bearer token required: diff --git a/telemetry-receiver/charts_ext.go b/telemetry-receiver/charts_ext.go index 320b351ab..6bc38d6d5 100644 --- a/telemetry-receiver/charts_ext.go +++ b/telemetry-receiver/charts_ext.go @@ -13,6 +13,7 @@ type extCharts struct { Distro barChart Arch barChart Nodes barChart + Deployment barChart Games barChart Features barChart VersionsInstall barChart @@ -56,6 +57,7 @@ func buildExtendedCharts(v Views, table bool) *extCharts { c.Distro = shareChart("Distribution", "Distribution", e.Env.Distro, false) c.Arch = shareChart("Architecture", "Architecture", e.Env.Arch, false) c.Nodes = shareChart("Cluster nodes", "Node band", e.Env.Nodes, false) + c.Deployment = shareChart("Deployment", "Deployment", e.Env.Deployment, false) games := make([]ShareRow, len(e.Games)) for i, g := range e.Games { @@ -95,7 +97,7 @@ func buildExtendedCharts(v Views, table bool) *extCharts { } c.VersionsInstall = newBarChart("Versions by install", "Version", "Installs", false, countBars(labels, counts)) c.Trend = buildTrendChart(e.VersionTrend, table) - for _, bc := range []*barChart{&c.K8s, &c.Distro, &c.Arch, &c.Nodes, &c.Games, &c.Features, &c.VersionsInstall} { + for _, bc := range []*barChart{&c.K8s, &c.Distro, &c.Arch, &c.Nodes, &c.Deployment, &c.Games, &c.Features, &c.VersionsInstall} { bc.Table = table } c.New.Table, c.Lapsed.Table = table, table diff --git a/telemetry-receiver/charts_test.go b/telemetry-receiver/charts_test.go index aff652c9f..74924c949 100644 --- a/telemetry-receiver/charts_test.go +++ b/telemetry-receiver/charts_test.go @@ -120,7 +120,7 @@ func seedExtendedDash(t *testing.T, s *server) { for n := 1; n <= 2; n++ { body, sig := newExtInstall(n).report(t, s.now(), func(r *telemetryschema.Report) { if n == 2 { - r.Ext.Env.Distro, r.Ext.Features.DB, r.Ext.Features.Clusters = "eks", "postgres", "2-3" + r.Ext.Env.Distro, r.Ext.Features.DB, r.Ext.Features.Clusters = []string{"eks"}, "postgres", "2-3" } }) if w := extSend(t, s, body, sig); w.Code != http.StatusNoContent { diff --git a/telemetry-receiver/design-export/MANIFEST.md b/telemetry-receiver/design-export/MANIFEST.md index 1406da68a..8e8312b6c 100644 --- a/telemetry-receiver/design-export/MANIFEST.md +++ b/telemetry-receiver/design-export/MANIFEST.md @@ -72,3 +72,25 @@ Method: - **JSON.** `JSON.stringify(Get("i0xK1E", {depth: 40, includePathGeometry: true}))` was checked equal to the written file: 111,299 UTF-16 code units, FNV-1a 710213898. The file has no `"..."` elisions. - **PNG.** `Export(["i0xK1E"], "png", …, {scale: 2})` gives 2880×7091. The frame height is 3545.0000000000005, so the export rounds it up by one pixel. - **Content checks.** Each string matched only `i0xK1E.json`: "Behind v0.3.0, on its own scale", "Installs by their last reported version" and "Reports by version". + +## Incremental export (2026-10-17): deployment card (spec 022 T122, FR-045, OD-13 to OD-16) + +| Node | Change | +|---|---| +| `i0xK1E` | New row `ISPZo` "Environment row 3" under the Architecture and Cluster nodes row. Its left column holds card `lAzgJ` "Deployment", a copy of the Architecture card with three `wPbIp` Share row instances (cluster 82%, standalone 15%, other 3%). The right column is an empty grid cell `h5Dkd`, so the environment grid stays in pairs. Caption: "Share of install-days by how the panel runs: on a cluster, or standalone. Other is installs on an older version that doesn't report it." The Kubernetes version and Distribution captions now describe sets (OD-15): "Share of install-days reporting each minor version across their managed clusters. Installs with several report each, so shares can add up to more than 100%." Distribution reads the same with "each detected distribution". | + +Design notes: + +- **Components.** No new component. + - The card keeps the environment card layout and tokens (`$surface/surface`, `$radius/3xl`, `$border`, padding 20, gap 14), and its rows are `wPbIp` instances. + - Bar widths follow the file's convention: share × the 432 px track. + - The "other" label uses `$foreground/muted`, like the Other rows of the neighbouring cards. +- **Audit.** The `pen-componentize` audit found no detached copies, replaced instances or repeated structures. Its hand-built control findings on `fximY`, `y49Ie` and `jy9Rn` predate this change. +- **Modes.** The frame stays in dark mode. Light mode was checked by switching the frame's theme for a preview and back. +- **Narrow layout.** Below 960 px the template's `grid-2` collapses to one column, and the card's SVG scales like its neighbours'. + +Method: + +- **JSON.** `JSON.stringify(Get("i0xK1E", {depth: 40, includePathGeometry: true}))` was checked equal to the written file: 113,413 UTF-16 code units, FNV-1a 1968389605. The file has no `"..."` elisions. +- **PNG.** `Export(["i0xK1E"], "png", …, {scale: 2})` gives 2880×7553. +- **Content checks.** Each string matched only `i0xK1E.json`: "doesn't report it" and "across their managed clusters". diff --git a/telemetry-receiver/design-export/json/i0xK1E.json b/telemetry-receiver/design-export/json/i0xK1E.json index e2dee4eff..bae4937ef 100644 --- a/telemetry-receiver/design-export/json/i0xK1E.json +++ b/telemetry-receiver/design-export/json/i0xK1E.json @@ -4681,7 +4681,7 @@ "fill": "$foreground/muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Share of install-days by reported minor version.", + "content": "Share of install-days reporting each minor version across their managed clusters. Installs with several report each, so shares can add up to more than 100%.", "fontFamily": "$typography/font-sans", "fontSize": 13, "fontWeight": "normal" @@ -4864,7 +4864,7 @@ "fill": "$foreground/muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Share of install-days by detected distribution.", + "content": "Share of install-days reporting each detected distribution across their managed clusters. Installs with several report each, so shares can add up to more than 100%.", "fontFamily": "$typography/font-sans", "fontSize": 13, "fontWeight": "normal" @@ -5435,6 +5435,155 @@ } ] }, + { + "type": "frame", + "id": "ISPZo", + "name": "Environment row 3", + "width": "fill_container", + "gap": 16, + "children": [ + { + "type": "frame", + "id": "lAzgJ", + "name": "Deployment", + "width": "fill_container", + "fill": "$surface/surface", + "cornerRadius": "$radius/3xl", + "stroke": "$border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 14, + "padding": 20, + "children": [ + { + "type": "frame", + "id": "WL8dq", + "name": "Titles", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "tKndJ", + "name": "Title", + "fill": "$foreground/foreground", + "content": "Deployment", + "fontFamily": "$typography/font-sans", + "fontSize": 15, + "fontWeight": "600" + }, + { + "type": "text", + "id": "n15dT", + "name": "Sub", + "fill": "$foreground/muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Share of install-days by how the panel runs: on a cluster, or standalone. Other is installs on an older version that doesn't report it.", + "fontFamily": "$typography/font-sans", + "fontSize": 13, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "pFEjb", + "name": "Bars", + "width": "fill_container", + "layout": "vertical", + "gap": 10, + "children": [ + { + "id": "MAjkd", + "type": "ref", + "ref": "wPbIp", + "name": "cluster", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "cluster", + "fill": "$foreground/foreground" + }, + "dMFUm": { + "width": 354, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "82%" + } + } + }, + { + "id": "bbQZg", + "type": "ref", + "ref": "wPbIp", + "name": "standalone", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "standalone", + "fill": "$foreground/foreground" + }, + "dMFUm": { + "width": 65, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "15%" + } + } + }, + { + "id": "Km3lN", + "type": "ref", + "ref": "wPbIp", + "name": "other", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "other", + "fill": "$foreground/muted" + }, + "dMFUm": { + "width": 13, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "3%" + } + } + } + ] + } + ] + }, + { + "type": "frame", + "id": "h5Dkd", + "name": "Empty grid cell", + "width": "fill_container", + "height": "fill_container" + } + ] + }, { "type": "frame", "id": "X93xNq", diff --git a/telemetry-receiver/design-export/screenshots/i0xK1E.png b/telemetry-receiver/design-export/screenshots/i0xK1E.png index 21f54a4bb..0fd13ef40 100644 Binary files a/telemetry-receiver/design-export/screenshots/i0xK1E.png and b/telemetry-receiver/design-export/screenshots/i0xK1E.png differ diff --git a/telemetry-receiver/ingest_deployment_test.go b/telemetry-receiver/ingest_deployment_test.go new file mode 100644 index 000000000..0ba39ac6d --- /dev/null +++ b/telemetry-receiver/ingest_deployment_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "fmt" + "net/http" + "testing" + "time" + + "github.com/GameplanePanel/gameplane/telemetryschema" +) + +// TestIngestSchema2WritesDeploymentAndOneRowPerSetValue checks that a schema 2 +// report writes a deployment row and one daily_dim row per value of each set. +func TestIngestSchema2WritesDeploymentAndOneRowPerSetValue(t *testing.T) { + s, _ := newExtServer(t, config{}, extStart) + body, sig := newExtInstall(1).report(t, extStart, func(r *telemetryschema.Report) { + r.Ext.Env = telemetryschema.Env{ + K8s: []string{"1.30", "1.31"}, Distro: []string{"eks", "k3s"}, Arch: []string{"amd64", "arm64"}, + Nodes: "4-10", Deployment: "standalone", + } + }) + if w := extSend(t, s, body, sig); w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204 (%s)", w.Code, w.Body) + } + const day = "2026-10-07" + for _, c := range []struct{ dim, value string }{ + {"deployment", "standalone"}, + {"k8s", "1.30"}, + {"k8s", "1.31"}, + {"distro", "eks"}, + {"distro", "k3s"}, + {"nodes", "4-10"}, + } { + n := countRows(t.Context(), t, s.store, + `SELECT installs FROM daily_dim WHERE day = ? AND dim = ? AND value = ?`, day, c.dim, c.value) + if n != 1 { + t.Errorf("daily_dim (%s, %s) installs = %d, want 1", c.dim, c.value, n) + } + } + if n := countRows(t.Context(), t, s.store, + `SELECT count(*) FROM daily_dim WHERE day = ? AND dim = 'k8s'`, day); n != 2 { + t.Errorf("k8s rows = %d, want 2 (one per value)", n) + } +} + +// TestIngestSchema1CountsDeploymentAsOther checks that a schema 1 report, +// which carries no deployment, is counted under deployment "other". +func TestIngestSchema1CountsDeploymentAsOther(t *testing.T) { + s, _ := newExtServer(t, config{}, extStart) + body, sig := newExtInstall(1).report(t, extStart, func(r *telemetryschema.Report) { + r.Ext.Schema = 1 + r.Ext.Env = telemetryschema.Env{K8s: []string{"1.29"}, Distro: []string{"kind"}, Arch: []string{"amd64"}, Nodes: "1"} + }) + if w := extSend(t, s, body, sig); w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want 204 (%s)", w.Code, w.Body) + } + const day = "2026-10-07" + for _, c := range []struct{ dim, value string }{ + {"deployment", "other"}, + {"k8s", "1.29"}, + {"distro", "kind"}, + } { + n := countRows(t.Context(), t, s.store, + `SELECT installs FROM daily_dim WHERE day = ? AND dim = ? AND value = ?`, day, c.dim, c.value) + if n != 1 { + t.Errorf("daily_dim (%s, %s) installs = %d, want 1", c.dim, c.value, n) + } + } +} + +// TestBuildViewsDeploymentShares checks that the deployment share is split +// between the two installs that reported it. +func TestBuildViewsDeploymentShares(t *testing.T) { + s, clock := newExtServer(t, config{activityExpiryDays: 31}, extStart) + a, aSig := newExtInstall(1).report(t, extStart, func(r *telemetryschema.Report) { + r.Ext.Env.Deployment = "standalone" + }) + b, bSig := newExtInstall(2).report(t, extStart, func(r *telemetryschema.Report) { + r.Ext.Env.Deployment = "cluster" + }) + for _, c := range []struct{ body, sig string }{{a, aSig}, {b, bSig}} { + if w := extSend(t, s, c.body, c.sig); w.Code != http.StatusNoContent { + t.Fatalf("status = %d (%s)", w.Code, w.Body) + } + } + *clock = extStart.Add(24 * time.Hour) + v, err := BuildViews(t.Context(), s.store, 7, *clock) + if err != nil { + t.Fatal(err) + } + if v.Extended == nil { + t.Fatal("Extended is nil") + } + want := []ShareRow{{"cluster", 0.5}, {"standalone", 0.5}} + if got := fmt.Sprint(v.Extended.Env.Deployment); got != fmt.Sprint(want) { + t.Fatalf("deployment = %v, want %v", v.Extended.Env.Deployment, want) + } +} diff --git a/telemetry-receiver/ingest_ext.go b/telemetry-receiver/ingest_ext.go index 200a1107a..f99e50e8f 100644 --- a/telemetry-receiver/ingest_ext.go +++ b/telemetry-receiver/ingest_ext.go @@ -17,15 +17,16 @@ import ( // Dimension names stored in daily_dim.dim (data-model.md, Receiver). const ( - dimK8s = "k8s" - dimDistro = "distro" - dimArch = "arch" - dimNodes = "nodes" - dimTunnel = "tunnel" - dimClusters = "clusters" - dimDB = "db" - dimLanguage = "language" - dimFeature = "feature" + dimK8s = "k8s" + dimDistro = "distro" + dimArch = "arch" + dimNodes = "nodes" + dimDeployment = "deployment" + dimTunnel = "tunnel" + dimClusters = "clusters" + dimDB = "db" + dimLanguage = "language" + dimFeature = "feature" ) // featureNames are the boolean features stored under dimFeature, in display @@ -259,11 +260,17 @@ func writeExtendedAggregates(ctx context.Context, tx *sql.Tx, r extRecord, isNew } e := r.Ext dims := []struct{ dim, value string }{ - {dimK8s, e.Env.K8s}, {dimDistro, e.Env.Distro}, {dimNodes, e.Env.Nodes}, + {dimNodes, e.Env.Nodes}, {dimDeployment, e.Env.Deployment}, {dimClusters, e.Features.Clusters}, {dimDB, e.Features.DB}, {dimLanguage, e.Features.Language}, } - for _, a := range e.Env.Arch { - dims = append(dims, struct{ dim, value string }{dimArch, a}) + // k8s, distro and arch are sets (OD-15): one row per value present. + for _, set := range []struct { + dim string + values []string + }{{dimK8s, e.Env.K8s}, {dimDistro, e.Env.Distro}, {dimArch, e.Env.Arch}} { + for _, v := range set.values { + dims = append(dims, struct{ dim, value string }{set.dim, v}) + } } for _, t := range e.Features.Tunnels { dims = append(dims, struct{ dim, value string }{dimTunnel, t}) diff --git a/telemetry-receiver/ingest_ext_test.go b/telemetry-receiver/ingest_ext_test.go index dfbb1a1eb..ab9b8e75f 100644 --- a/telemetry-receiver/ingest_ext_test.go +++ b/telemetry-receiver/ingest_ext_test.go @@ -50,7 +50,7 @@ func (in extInstall) report(t *testing.T, sentAt time.Time, mod func(*telemetrys } rep := telemetryschema.Report{Version: "1.0.0", Servers: 3, Templates: 2, Ext: &telemetryschema.Extended{ Schema: telemetryschema.SchemaVersion, InstallID: in.id, - Env: telemetryschema.Env{K8s: "1.31", Distro: "k3s", Arch: []string{"amd64"}, Nodes: "1"}, + Env: telemetryschema.Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64"}, Nodes: "1", Deployment: "cluster"}, Games: telemetryschema.Games{Official: map[string]int{"minecraft-java": 2}, Custom: 1}, Features: telemetryschema.Features{ Backups: true, Tunnels: []string{"playit"}, Clusters: "1", DB: "sqlite", Language: "en", @@ -325,7 +325,7 @@ func TestIngestExtendedStructuralErrorIs400(t *testing.T) { in := newExtInstall(1) before := extDump(t, s.store) for name, mod := range map[string]func(*telemetryschema.Report){ - "schema 2": func(r *telemetryschema.Report) { r.Ext.Schema = 2 }, + "schema 3": func(r *telemetryschema.Report) { r.Ext.Schema = 3 }, "no architecture": func(r *telemetryschema.Report) { r.Ext.Env.Arch = nil }, "bad key": func(r *telemetryschema.Report) { r.Ext.Key = "short" }, } { diff --git a/telemetry-receiver/population_test.go b/telemetry-receiver/population_test.go index d63a2390c..122da5944 100644 --- a/telemetry-receiver/population_test.go +++ b/telemetry-receiver/population_test.go @@ -331,7 +331,7 @@ func TestBuildViewsExtendedSharesGamesAndFeatures(t *testing.T) { r.Ext.Games = telemetryschema.Games{Official: map[string]int{"minecraft-java": 4}, Custom: 0} }) b, bSig := newExtInstall(2).report(t, extStart, func(r *telemetryschema.Report) { - r.Ext.Env.Distro, r.Ext.Env.Nodes, r.Ext.Env.K8s = "eks", "2-3", "1.30" + r.Ext.Env.Distro, r.Ext.Env.Nodes, r.Ext.Env.K8s = []string{"eks"}, "2-3", []string{"1.30"} r.Ext.Features.Backups, r.Ext.Features.Tunnels = false, []string{} r.Ext.Features.DB, r.Ext.Features.Clusters = "postgres", "2-3" r.Ext.Games = telemetryschema.Games{Official: map[string]int{"terraria": 1}, Custom: 2} diff --git a/telemetry-receiver/specs.md b/telemetry-receiver/specs.md index 626e54767..a9d64c672 100644 --- a/telemetry-receiver/specs.md +++ b/telemetry-receiver/specs.md @@ -132,7 +132,7 @@ All series are in memory and reset on restart; the durable figures are the aggre ## Key invariants -- **Stores daily aggregates and expiring activity records only, never raw reports or addresses**: the schema (`meta`, `daily_basic`, `daily_version`, `daily_fleet`, `daily_ext`, `daily_dim`, `daily_game`, `daily_install_version`, `daily_version_change`, `activity`) holds counts, bands and categories per UTC day, plus one activity record per install keyed by `HMAC-SHA256(pepper, installID)`. Source addresses live in limiter memory only and are never written to the store or the logs. No extended attribute is stored next to an install ID. The version trend (spec 022 FR-042) keeps only daily counts: installs per reported version, and installs that upgraded, downgraded or changed version without a semver order. No version history is linked to an install ID. +- **Stores daily aggregates and expiring activity records only, never raw reports or addresses**: the schema (`meta`, `daily_basic`, `daily_version`, `daily_fleet`, `daily_ext`, `daily_dim`, `daily_game`, `daily_install_version`, `daily_version_change`, `activity`) holds counts, bands and categories per UTC day, plus one activity record per install keyed by `HMAC-SHA256(pepper, installID)`. Source addresses live in limiter memory only and are never written to the store or the logs. No extended attribute is stored next to an install ID. The version trend (spec 022 FR-042) keeps only daily counts: installs per reported version, and installs that upgraded, downgraded or changed version without a semver order. No version history is linked to an install ID. Since spec 022 FR-045, `daily_dim` also holds the deployment (`cluster`, `standalone`, or `other` for schema 1 reports), and one row per Kubernetes minor version and distribution an install reports (OD-15). - **Basic or extended report, both strictly validated**: a body is accepted only when it decodes as exactly one basic report or one basic report with a valid `ext` part; anything else is `400`. Out-of-set values in extended categories fold to `other`. - **Extended reports must be signed**: a report with `ext` and a missing or invalid signature is refused (`403 bad_signature`) before any write, and an install ID is bound to the first valid key that claims it. - **Refusal invariant (dashboard)**: an unauthenticated dashboard response never contains a figure, a date other than page chrome, a version, category or module name, or anything that varies with whether data exists. A wrong token gets the same "Invalid credentials" login page whether or not the receiver is empty. diff --git a/telemetry-receiver/telemetry-dashboard.pen b/telemetry-receiver/telemetry-dashboard.pen index 33284f55c..9ab6d5dd8 100644 --- a/telemetry-receiver/telemetry-dashboard.pen +++ b/telemetry-receiver/telemetry-dashboard.pen @@ -1,5 +1,5 @@ { - "version": "2.20", + "version": "2.21", "children": [ { "type": "frame", @@ -17292,7 +17292,7 @@ "fill": "$foreground/muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Share of install-days by reported minor version.", + "content": "Share of install-days reporting each minor version across their managed clusters. Installs with several report each, so shares can add up to more than 100%.", "fontFamily": "$typography/font-sans", "fontSize": 13, "fontWeight": "normal" @@ -17475,7 +17475,7 @@ "fill": "$foreground/muted", "textGrowth": "fixed-width", "width": "fill_container", - "content": "Share of install-days by detected distribution.", + "content": "Share of install-days reporting each detected distribution across their managed clusters. Installs with several report each, so shares can add up to more than 100%.", "fontFamily": "$typography/font-sans", "fontSize": 13, "fontWeight": "normal" @@ -18046,6 +18046,155 @@ } ] }, + { + "type": "frame", + "id": "ISPZo", + "name": "Environment row 3", + "width": "fill_container", + "gap": 16, + "children": [ + { + "type": "frame", + "id": "lAzgJ", + "name": "Deployment", + "width": "fill_container", + "fill": "$surface/surface", + "cornerRadius": "$radius/3xl", + "stroke": "$border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 14, + "padding": 20, + "children": [ + { + "type": "frame", + "id": "WL8dq", + "name": "Titles", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "tKndJ", + "name": "Title", + "fill": "$foreground/foreground", + "content": "Deployment", + "fontFamily": "$typography/font-sans", + "fontSize": 15, + "fontWeight": "600" + }, + { + "type": "text", + "id": "n15dT", + "name": "Sub", + "fill": "$foreground/muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Share of install-days by how the panel runs: on a cluster, or standalone. Other is installs on an older version that doesn't report it.", + "fontFamily": "$typography/font-sans", + "fontSize": 13, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "pFEjb", + "name": "Bars", + "width": "fill_container", + "layout": "vertical", + "gap": 10, + "children": [ + { + "id": "MAjkd", + "type": "ref", + "ref": "wPbIp", + "name": "cluster", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "cluster", + "fill": "$foreground/foreground" + }, + "dMFUm": { + "width": 354, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "82%" + } + } + }, + { + "id": "bbQZg", + "type": "ref", + "ref": "wPbIp", + "name": "standalone", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "standalone", + "fill": "$foreground/foreground" + }, + "dMFUm": { + "width": 65, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "15%" + } + } + }, + { + "id": "Km3lN", + "type": "ref", + "ref": "wPbIp", + "name": "other", + "width": "fill_container", + "descendants": { + "j1J75": { + "content": "other", + "fill": "$foreground/muted" + }, + "dMFUm": { + "width": 13, + "cornerRadius": [ + 0, + 4, + 4, + 0 + ] + }, + "uTqdm": { + "content": "3%" + } + } + } + ] + } + ] + }, + { + "type": "frame", + "id": "h5Dkd", + "name": "Empty grid cell", + "width": "fill_container", + "height": "fill_container" + } + ] + }, { "type": "frame", "id": "X93xNq", diff --git a/telemetry-receiver/views.go b/telemetry-receiver/views.go index 8a19bc2cc..88bafb206 100644 --- a/telemetry-receiver/views.go +++ b/telemetry-receiver/views.go @@ -111,10 +111,11 @@ type VersionTrendDay struct { // EnvViews are the environment shares. type EnvViews struct { - K8s []ShareRow `json:"k8s"` - Distro []ShareRow `json:"distro"` - Arch []ShareRow `json:"arch"` - Nodes []ShareRow `json:"nodes"` + K8s []ShareRow `json:"k8s"` + Distro []ShareRow `json:"distro"` + Arch []ShareRow `json:"arch"` + Nodes []ShareRow `json:"nodes"` + Deployment []ShareRow `json:"deployment"` } // ShareRow is a category's share of the reporting install-days in the range @@ -445,10 +446,11 @@ func buildExtended(ctx context.Context, st *store, rangeDays int, from, to strin return nil, err } e.Env = EnvViews{ - K8s: sortedShares(dims[dimK8s], nil), - Distro: sortedShares(dims[dimDistro], nil), - Arch: sortedShares(dims[dimArch], nil), - Nodes: sortedShares(dims[dimNodes], telemetryschema.NodeBands), + K8s: sortedShares(dims[dimK8s], nil), + Distro: sortedShares(dims[dimDistro], nil), + Arch: sortedShares(dims[dimArch], nil), + Nodes: sortedShares(dims[dimNodes], telemetryschema.NodeBands), + Deployment: sortedShares(dims[dimDeployment], nil), } e.Tunnels = sortedShares(dims[dimTunnel], nil) e.Clusters = sortedShares(dims[dimClusters], telemetryschema.ClusterBands) diff --git a/telemetry-receiver/web/overview_extended.html b/telemetry-receiver/web/overview_extended.html index 8908a38f6..f649cf11f 100644 --- a/telemetry-receiver/web/overview_extended.html +++ b/telemetry-receiver/web/overview_extended.html @@ -35,11 +35,11 @@

Extended reports

-
Kubernetes versionShare of install-days by reported minor version.
+
Kubernetes versionShare of install-days reporting each minor version across their managed clusters. Installs with several report each, so shares can add up to more than 100%.
{{template "bars" .K8s}}
-
DistributionShare of install-days by detected distribution.
+
DistributionShare of install-days reporting each detected distribution across their managed clusters. Installs with several report each, so shares can add up to more than 100%.
{{template "bars" .Distro}}
@@ -55,6 +55,12 @@

Extended reports

+
DeploymentShare of install-days by how the panel runs: on a cluster, or standalone. Other is installs on an older version that doesn't report it.
+{{template "bars" .Deployment}} +
+
+
+
GamesShare of install-days running at least one server of each official module. Uploaded and third-party modules count as Custom.
{{template "bars" .Games}}
diff --git a/telemetryschema/decode.go b/telemetryschema/decode.go index bedc9ad0e..f4794ac9f 100644 --- a/telemetryschema/decode.go +++ b/telemetryschema/decode.go @@ -25,7 +25,7 @@ var ErrInvalidPayload = errors.New("invalid payload") var ErrUnsupportedSchema = fmt.Errorf("%w: unsupported ext schema", ErrInvalidPayload) // SchemaVersion is the extended-tier schema this package reads and writes. -const SchemaVersion = 1 +const SchemaVersion = 2 // keySize is the length in bytes of an Ed25519 public key. const keySize = 32 @@ -270,13 +270,13 @@ func decodeExt(raw json.RawMessage) (*Extended, error) { return nil, err } var e Extended - if e.Schema, err = asCount(o.m["schema"], "ext.schema", SchemaVersion); err != nil { + if e.Schema, err = asCount(o.m["schema"], "ext.schema", 1); err != nil { return nil, err } if e.InstallID, err = asString(o.m["installId"], "ext.installId"); err != nil { return nil, err } - if e.Env, err = decodeEnv(o.m["env"]); err != nil { + if e.Env, err = decodeEnv(o.m["env"], e.Schema); err != nil { return nil, err } if e.Games, err = decodeGames(o.m["games"]); err != nil { @@ -307,32 +307,53 @@ func decodeExt(raw json.RawMessage) (*Extended, error) { return &e, nil } -// decodeEnv validates ext.env. -func decodeEnv(raw json.RawMessage) (Env, error) { +// decodeEnv validates ext.env. Schema 1 carries k8s and distro as single +// strings and no deployment: they decode as one-value sets, and Deployment is +// "other" (OD-14). Schema 2 carries k8s and distro as non-empty sets and a +// required deployment (OD-13, OD-15). nodes is one band in both. +func decodeEnv(raw json.RawMessage, schema int) (Env, error) { o, err := readNested(raw, "ext.env") if err != nil { return Env{}, err } - if err := o.expect([]string{"k8s", "distro", "arch", "nodes"}, nil); err != nil { - return Env{}, err + keys := []string{"k8s", "distro", "arch", "nodes"} + if schema >= 2 { + keys = append(keys, "deployment") } - var e Env - k8s, err := asString(o.m["k8s"], "ext.env.k8s") - if err != nil { + if err := o.expect(keys, nil); err != nil { return Env{}, err } - e.K8s = other - if K8sMinorRE.MatchString(k8s) { - e.K8s = k8s - } - if e.Distro, err = asEnum(o.m["distro"], "ext.env.distro", Distros); err != nil { - return Env{}, err + var e Env + if schema >= 2 { + if e.K8s, err = asK8sSet(o.m["k8s"]); err != nil { + return Env{}, err + } + if e.Distro, err = asEnumSet(o.m["distro"], "ext.env.distro", Distros); err != nil { + return Env{}, err + } + if e.Deployment, err = asEnum(o.m["deployment"], "ext.env.deployment", Deployments); err != nil { + return Env{}, err + } + } else { + var k8s, distro string + if k8s, err = asString(o.m["k8s"], "ext.env.k8s"); err != nil { + return Env{}, err + } + if distro, err = asEnum(o.m["distro"], "ext.env.distro", Distros); err != nil { + return Env{}, err + } + e.K8s, e.Distro, e.Deployment = []string{k8sMinor(k8s)}, []string{distro}, other } if e.Arch, err = asEnumSet(o.m["arch"], "ext.env.arch", Arches); err != nil { return Env{}, err } - if len(e.Arch) == 0 { - return Env{}, fmt.Errorf("%w: ext.env.arch must not be empty", ErrInvalidPayload) + for _, f := range []struct { + name string + set []string + }{{"k8s", e.K8s}, {"distro", e.Distro}, {"arch", e.Arch}} { + if len(f.set) == 0 { + return Env{}, fmt.Errorf("%w: ext.env.%s must not be empty", ErrInvalidPayload, f.name) + } } if e.Nodes, err = asEnum(o.m["nodes"], "ext.env.nodes", NodeBands); err != nil { return Env{}, err @@ -340,6 +361,33 @@ func decodeEnv(raw json.RawMessage) (Env, error) { return e, nil } +// k8sMinor returns v when it is a 1. version and "other" otherwise. +func k8sMinor(v string) string { + if K8sMinorRE.MatchString(v) { + return v + } + return other +} + +// asK8sSet decodes schema 2's ext.env.k8s: an array of strings, each folded by +// k8sMinor, returned sorted and de-duplicated and never nil. An element that is +// null or not a string is invalid. +func asK8sSet(raw json.RawMessage) ([]string, error) { + var in []*string + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("%w: ext.env.k8s must be an array of strings: %w", ErrInvalidPayload, err) + } + out := make([]string, 0, len(in)) + for _, p := range in { + if p == nil { + return nil, fmt.Errorf("%w: ext.env.k8s must not contain null", ErrInvalidPayload) + } + out = append(out, k8sMinor(*p)) + } + sort.Strings(out) + return slices.Compact(out), nil +} + // decodeGames validates ext.games. Counts of modules outside the catalog are // added to Custom. func decodeGames(raw json.RawMessage) (Games, error) { diff --git a/telemetryschema/decode_schema2_test.go b/telemetryschema/decode_schema2_test.go new file mode 100644 index 000000000..e9501104d --- /dev/null +++ b/telemetryschema/decode_schema2_test.go @@ -0,0 +1,204 @@ +package telemetryschema + +import ( + "errors" + "reflect" + "strings" + "testing" + "time" +) + +// validExt2 is validExt on schema 2: k8s and distro are sets and the +// deployment is set. +var validExt2 = strings.Replace(strings.Replace(validExt, `"schema":1,`, `"schema":2,`, 1), + `"env":{"k8s":"1.31","distro":"k3s","arch":["amd64"],"nodes":"1"}`, + `"env":{"k8s":["1.31","1.30","1.31"],"distro":["k3s","eks"],"arch":["amd64"],"nodes":"4-10","deployment":"standalone"}`, 1) + +// mut2 replaces old with replacement in validExt2 and panics when old is +// absent, so a stale test fixture fails loudly instead of silently testing +// nothing. +func mut2(old, replacement string) string { + if !strings.Contains(validExt2, old) { + panic("schema 2 fixture does not contain " + old) + } + return strings.Replace(validExt2, old, replacement, 1) +} + +// TestDecodeExtSchema2Valid decodes the schema 2 fixture: the sets come back +// sorted and de-duplicated, and the deployment is kept. +func TestDecodeExtSchema2Valid(t *testing.T) { + t.Parallel() + r, info, err := Decode(withExt(validExt2)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if info.ExtDropped { + t.Fatal("ExtDropped = true for a valid schema 2 ext") + } + if r.Ext == nil { + t.Fatal("Ext is nil") + } + if r.Ext.Schema != 2 { + t.Fatalf("Schema = %d, want 2", r.Ext.Schema) + } + want := Env{K8s: []string{"1.30", "1.31"}, Distro: []string{"eks", "k3s"}, Arch: []string{"amd64"}, Nodes: "4-10", Deployment: "standalone"} + if !reflect.DeepEqual(r.Ext.Env, want) { + t.Fatalf("Env = %+v, want %+v", r.Ext.Env, want) + } +} + +// TestDecodeExtSchema2Folds checks that out-of-set values in the schema 2 +// sets and deployment fold to "other", and that known values are kept. +func TestDecodeExtSchema2Folds(t *testing.T) { + t.Parallel() + cases := []struct { + name string + ext string + got func(Env) any + want any + }{ + {"k8s folds out-of-set members", mut2(`["1.31","1.30","1.31"]`, `["v1.31","1.29"]`), func(e Env) any { return e.K8s }, []string{"1.29", "other"}}, + {"distro folds unknown", mut2(`["k3s","eks"]`, `["mystery"]`), func(e Env) any { return e.Distro }, []string{"other"}}, + {"deployment folds unknown", mut2(`"deployment":"standalone"`, `"deployment":"edge"`), func(e Env) any { return e.Deployment }, "other"}, + {"deployment cluster kept", mut2(`"deployment":"standalone"`, `"deployment":"cluster"`), func(e Env) any { return e.Deployment }, "cluster"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + r, _, err := Decode(withExt(tc.ext)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if got := tc.got(r.Ext.Env); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %#v, want %#v", got, tc.want) + } + }) + } +} + +// TestDecodeExtSchema2Rejects covers the schema 2 env rules: deployment is +// required, the sets are non-empty arrays of strings, and no unknown keys are +// allowed. None of these is an unsupported-schema error. +func TestDecodeExtSchema2Rejects(t *testing.T) { + t.Parallel() + cases := map[string]string{ + "missing deployment": mut2(`,"deployment":"standalone"`, ``), + "empty k8s": mut2(`["1.31","1.30","1.31"]`, `[]`), + "empty distro": mut2(`["k3s","eks"]`, `[]`), + "k8s as a string": mut2(`["1.31","1.30","1.31"]`, `"1.31"`), + "distro as a string": mut2(`["k3s","eks"]`, `"k3s"`), + "k8s null member": mut2(`["1.31","1.30","1.31"]`, `[null]`), + "deployment number": mut2(`"deployment":"standalone"`, `"deployment":1`), + "unknown env key": mut2(`"nodes":"4-10"`, `"nodes":"4-10","extra":1`), + } + for name, ext := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + got, info, err := Decode(withExt(ext)) + if err == nil { + t.Fatalf("expected error, got %+v (info %+v)", got, info) + } + if !errors.Is(err, ErrInvalidPayload) { + t.Fatalf("error chain missing ErrInvalidPayload: %v", err) + } + if errors.Is(err, ErrUnsupportedSchema) { + t.Fatalf("error must not be ErrUnsupportedSchema: %v", err) + } + }) + } +} + +// TestDecodeExtSchema1RefusesDeployment checks that a schema 1 env may not +// carry a deployment: the field belongs to schema 2. +func TestDecodeExtSchema1RefusesDeployment(t *testing.T) { + t.Parallel() + ext := mut(`"nodes":"1"}`, `"nodes":"1","deployment":"cluster"}`) + _, _, err := Decode(withExt(ext)) + if !errors.Is(err, ErrInvalidPayload) { + t.Fatalf("err = %v, want ErrInvalidPayload", err) + } +} + +// TestDecodeExtSchema1DecodesAsOneValueSets checks that a schema 1 report +// decodes k8s and distro as one-value sets and its deployment as "other" +// (OD-14). +func TestDecodeExtSchema1DecodesAsOneValueSets(t *testing.T) { + t.Parallel() + r, _, err := Decode(withExt(validExt)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if r.Ext == nil { + t.Fatal("Ext is nil") + } + want := Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64"}, Nodes: "1", Deployment: "other"} + if !reflect.DeepEqual(r.Ext.Env, want) { + t.Fatalf("Env = %+v, want %+v", r.Ext.Env, want) + } +} + +// TestEncodeSchema2RoundTrip checks that a schema 2 report encodes its sets +// sorted and that Decode returns the same environment. +func TestEncodeSchema2RoundTrip(t *testing.T) { + t.Parallel() + r := Report{ + Version: "0.3.0", Servers: 3, Templates: 7, + Ext: &Extended{ + Schema: 2, + InstallID: testID, + Env: Env{ + K8s: []string{"1.31", "1.30"}, Distro: []string{"k3s", "eks"}, Arch: []string{"amd64"}, + Nodes: "1", Deployment: "cluster", + }, + Games: Games{Official: map[string]int{"minecraft-java": 2}, Custom: 1}, + Features: Features{Tunnels: []string{"playit"}, Clusters: "1", DB: "sqlite", Language: "en"}, + Key: testKey, + SentAt: time.Date(2026, 10, 6, 9, 12, 44, 0, time.UTC), + }, + } + body, err := Encode(r) + if err != nil { + t.Fatalf("Encode: %v", err) + } + const frag = `"env":{"k8s":["1.30","1.31"],"distro":["eks","k3s"],"arch":["amd64"],"nodes":"1","deployment":"cluster"}` + if !strings.Contains(string(body), frag) { + t.Fatalf("body %s does not contain %s", body, frag) + } + back, info, err := Decode(body) + if err != nil { + t.Fatalf("Decode(Encode(r)): %v", err) + } + if info.ExtDropped { + t.Fatal("ExtDropped = true for an encoded schema 2 report") + } + want := Env{K8s: []string{"1.30", "1.31"}, Distro: []string{"eks", "k3s"}, Arch: []string{"amd64"}, Nodes: "1", Deployment: "cluster"} + if !reflect.DeepEqual(back.Ext.Env, want) { + t.Fatalf("Env = %+v, want %+v", back.Ext.Env, want) + } +} + +// TestEncodeSchema1WritesSingleValues checks that a schema 1 report writes +// k8s and distro as single values and no deployment, folding a multi-value +// set to "other". +func TestEncodeSchema1WritesSingleValues(t *testing.T) { + t.Parallel() + r := Report{Ext: &Extended{ + Schema: 1, + InstallID: testID, + Env: Env{ + K8s: []string{"1.31", "1.30"}, Distro: []string{"k3s"}, Arch: []string{"amd64"}, + Nodes: "1", Deployment: "cluster", + }, + }} + body, err := Encode(r) + if err != nil { + t.Fatalf("Encode: %v", err) + } + const frag = `"env":{"k8s":"other","distro":"k3s","arch":["amd64"],"nodes":"1"}` + if !strings.Contains(string(body), frag) { + t.Fatalf("body %s does not contain %s", body, frag) + } + if strings.Contains(string(body), "deployment") { + t.Fatalf("schema 1 body must not carry a deployment: %s", body) + } +} diff --git a/telemetryschema/decode_test.go b/telemetryschema/decode_test.go index 8276f7013..884e46858 100644 --- a/telemetryschema/decode_test.go +++ b/telemetryschema/decode_test.go @@ -217,7 +217,7 @@ func TestDecodeExtValid(t *testing.T) { want := &Extended{ Schema: 1, InstallID: testID, - Env: Env{K8s: "1.31", Distro: "k3s", Arch: []string{"amd64"}, Nodes: "1"}, + Env: Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64"}, Nodes: "1", Deployment: "other"}, Games: Games{Official: map[string]int{"minecraft-java": 2, "terraria": 1}, Custom: 0}, Features: Features{ WakeOnConnect: true, Tunnels: []string{"playit"}, Backups: true, @@ -384,16 +384,16 @@ func TestDecodeExtFoldsOutOfSetValues(t *testing.T) { got func(*Extended) any want any }{ - {"known distro kept", mut(`"k3s"`, `"talos"`), func(e *Extended) any { return e.Env.Distro }, "talos"}, - {"unknown distro", mut(`"k3s"`, `"mystery"`), func(e *Extended) any { return e.Env.Distro }, "other"}, - {"wrong-case distro", mut(`"k3s"`, `"K3S"`), func(e *Extended) any { return e.Env.Distro }, "other"}, - {"empty distro", mut(`"k3s"`, `""`), func(e *Extended) any { return e.Env.Distro }, "other"}, - {"k8s minor kept", mut(`"1.31"`, `"1.123"`), func(e *Extended) any { return e.Env.K8s }, "1.123"}, - {"k8s with v prefix", mut(`"1.31"`, `"v1.31"`), func(e *Extended) any { return e.Env.K8s }, "other"}, - {"k8s with patch", mut(`"1.31"`, `"1.31.2"`), func(e *Extended) any { return e.Env.K8s }, "other"}, - {"k8s major 2", mut(`"1.31"`, `"2.0"`), func(e *Extended) any { return e.Env.K8s }, "other"}, - {"k8s four digit minor", mut(`"1.31"`, `"1.1234"`), func(e *Extended) any { return e.Env.K8s }, "other"}, - {"k8s empty", mut(`"1.31"`, `""`), func(e *Extended) any { return e.Env.K8s }, "other"}, + {"known distro kept", mut(`"k3s"`, `"talos"`), func(e *Extended) any { return e.Env.Distro }, []string{"talos"}}, + {"unknown distro", mut(`"k3s"`, `"mystery"`), func(e *Extended) any { return e.Env.Distro }, []string{"other"}}, + {"wrong-case distro", mut(`"k3s"`, `"K3S"`), func(e *Extended) any { return e.Env.Distro }, []string{"other"}}, + {"empty distro", mut(`"k3s"`, `""`), func(e *Extended) any { return e.Env.Distro }, []string{"other"}}, + {"k8s minor kept", mut(`"1.31"`, `"1.123"`), func(e *Extended) any { return e.Env.K8s }, []string{"1.123"}}, + {"k8s with v prefix", mut(`"1.31"`, `"v1.31"`), func(e *Extended) any { return e.Env.K8s }, []string{"other"}}, + {"k8s with patch", mut(`"1.31"`, `"1.31.2"`), func(e *Extended) any { return e.Env.K8s }, []string{"other"}}, + {"k8s major 2", mut(`"1.31"`, `"2.0"`), func(e *Extended) any { return e.Env.K8s }, []string{"other"}}, + {"k8s four digit minor", mut(`"1.31"`, `"1.1234"`), func(e *Extended) any { return e.Env.K8s }, []string{"other"}}, + {"k8s empty", mut(`"1.31"`, `""`), func(e *Extended) any { return e.Env.K8s }, []string{"other"}}, {"arch sorted and de-duplicated", mut(`["amd64"]`, `["riscv64","arm64","amd64","arm64"]`), func(e *Extended) any { return e.Env.Arch }, []string{"amd64", "arm64", "riscv64"}}, {"arch unknown folds and collapses", mut(`["amd64"]`, `["x86","amd64","sparc","other"]`), func(e *Extended) any { return e.Env.Arch }, []string{"amd64", "other"}}, {"arch only unknown", mut(`["amd64"]`, `["x86"]`), func(e *Extended) any { return e.Env.Arch }, []string{"other"}}, @@ -515,12 +515,12 @@ func TestDecodeExtDropsMalformedInstallID(t *testing.T) { func TestDecodeExtSchema(t *testing.T) { t.Parallel() for name, ext := range map[string]string{ - "schema 2": setAt(t, 2, "schema"), + "schema 4": setAt(t, 4, "schema"), "schema 3": setAt(t, 3, "schema"), "schema 1000": setAt(t, 1000, "schema"), - "schema 2 unknown keys": mut(`"schema":1,`, `"schema":2,"newField":{"x":1},`), - "schema 2 missing keys": `{"schema":2}`, - "schema 2 bad fields": strings.Replace(setAt(t, 5, "env"), `"schema":1`, `"schema":2`, 1), + "schema 3 unknown keys": mut(`"schema":1,`, `"schema":3,"newField":{"x":1},`), + "schema 3 missing keys": `{"schema":3}`, + "schema 3 bad fields": strings.Replace(setAt(t, 5, "env"), `"schema":1`, `"schema":3`, 1), } { t.Run(name, func(t *testing.T) { _, _, err := Decode(withExt(ext)) @@ -571,7 +571,7 @@ func TestEncodeDeterministicRoundTrip(t *testing.T) { Ext: &Extended{ Schema: 1, InstallID: testID, - Env: Env{K8s: "1.31", Distro: "k3s", Arch: []string{"arm64", "amd64", "arm64"}, Nodes: "1"}, + Env: Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"arm64", "amd64", "arm64"}, Nodes: "1"}, Games: Games{Official: map[string]int{"terraria": 1, "minecraft-java": 2}, Custom: 0}, Features: Features{ WakeOnConnect: true, Tunnels: []string{"playit", "frp"}, Backups: true, @@ -620,7 +620,7 @@ func TestEncodeDeterministicRoundTrip(t *testing.T) { back.Ext.SentAt = time.Time{} wantExt := &Extended{ Schema: 1, InstallID: testID, - Env: Env{K8s: "1.31", Distro: "k3s", Arch: []string{"amd64", "arm64"}, Nodes: "1"}, + Env: Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64", "arm64"}, Nodes: "1", Deployment: "other"}, Games: Games{Official: map[string]int{"minecraft-java": 2, "terraria": 1}}, Features: Features{ WakeOnConnect: true, Tunnels: []string{"frp", "playit"}, Backups: true, diff --git a/telemetryschema/enums.go b/telemetryschema/enums.go index 3baf5decd..4990e51e8 100644 --- a/telemetryschema/enums.go +++ b/telemetryschema/enums.go @@ -26,6 +26,9 @@ var ( Distros = []string{"k3s", "rke2", "k0s", "eks", "gke", "aks", "openshift", "microk8s", "minikube", "kind", "doks", "talos", "other"} // Arches lists the recognised CPU architectures. Arches = []string{"amd64", "arm64", "arm", "ppc64le", "s390x", "riscv64", "other"} + // Deployments lists how an install runs: on its own cluster, or as a + // standalone panel with no local cluster (FR-045). + Deployments = []string{"cluster", "standalone"} // Tunnels lists the recognised tunnel providers. Tunnels = []string{"frp", "tailscale", "playit", "other"} // DBs lists the recognised API database drivers. diff --git a/telemetryschema/report.go b/telemetryschema/report.go index 735afab45..ca5896e39 100644 --- a/telemetryschema/report.go +++ b/telemetryschema/report.go @@ -32,11 +32,16 @@ type Extended struct { } // Env describes the install's environment, using only enumerated values. +// From schema 2 it covers every reachable managed cluster: K8s and Distro are +// sets like Arch, and Deployment says whether the panel runs on a cluster or +// standalone. A schema 1 report decodes K8s and Distro as one-value sets and +// Deployment as "other". type Env struct { - K8s string `json:"k8s"` - Distro string `json:"distro"` - Arch []string `json:"arch"` - Nodes string `json:"nodes"` + K8s []string `json:"k8s"` + Distro []string `json:"distro"` + Arch []string `json:"arch"` + Nodes string `json:"nodes"` + Deployment string `json:"deployment"` } // Games counts local GameServers: Official maps an official catalog module @@ -60,17 +65,35 @@ type Features struct { } // wireExtended is Extended as it appears on the wire: SentAt is a -// second-precision RFC 3339 UTC string. +// second-precision RFC 3339 UTC string. Env is a wireEnv1 or a wireEnv2, by Schema. type wireExtended struct { Schema int `json:"schema"` InstallID string `json:"installId"` - Env Env `json:"env"` + Env any `json:"env"` Games Games `json:"games"` Features Features `json:"features"` Key string `json:"key"` SentAt string `json:"sentAt"` } +// wireEnv1 is Env as schema 1 writes it: k8s and distro are single values and +// there is no deployment. +type wireEnv1 struct { + K8s string `json:"k8s"` + Distro string `json:"distro"` + Arch []string `json:"arch"` + Nodes string `json:"nodes"` +} + +// wireEnv2 is Env as schema 2 writes it (OD-13, OD-15). +type wireEnv2 struct { + K8s []string `json:"k8s"` + Distro []string `json:"distro"` + Arch []string `json:"arch"` + Nodes string `json:"nodes"` + Deployment string `json:"deployment"` +} + // wireReport is Report as it appears on the wire. type wireReport struct { Version string `json:"version"` @@ -82,7 +105,7 @@ type wireReport struct { // Encode returns the canonical JSON body for r. The output is deterministic: // arrays are sorted and de-duplicated, object keys are sorted (encoding/json // sorts map keys), nil arrays and maps are written as [] and {} (never null), -// and SentAt is written in UTC at second precision. Encode does not fold +// and SentAt is written in UTC at second precision. Schema 1 writes env.k8s and env.distro as single values and no deployment; later schemas write them as sets with the deployment. Encode does not fold // out-of-set values; Decode does. Negative counts are rejected with // ErrInvalidPayload. func Encode(r Report) ([]byte, error) { @@ -95,8 +118,13 @@ func Encode(r Report) ([]byte, error) { for k, v := range e.Games.Official { official[k] = v } - env := e.Env - env.Arch = sortedSet(env.Arch) + var env any = wireEnv2{ + K8s: sortedSet(e.Env.K8s), Distro: sortedSet(e.Env.Distro), Arch: sortedSet(e.Env.Arch), + Nodes: e.Env.Nodes, Deployment: e.Env.Deployment, + } + if e.Schema < 2 { + env = wireEnv1{K8s: single(e.Env.K8s), Distro: single(e.Env.Distro), Arch: sortedSet(e.Env.Arch), Nodes: e.Env.Nodes} + } feat := e.Features feat.Tunnels = sortedSet(feat.Tunnels) w.Ext = &wireExtended{ @@ -126,3 +154,16 @@ func sortedSet(in []string) []string { sort.Strings(out) return slices.Compact(out) } + +// single returns the one value of a set for a schema 1 field: "" for an +// empty set, the value for a one-value set, and "other" for more than one. +func single(set []string) string { + switch s := sortedSet(set); len(s) { + case 0: + return "" + case 1: + return s[0] + default: + return other + } +} diff --git a/telemetryschema/sign_test.go b/telemetryschema/sign_test.go index d3e6cc54f..d00a07a68 100644 --- a/telemetryschema/sign_test.go +++ b/telemetryschema/sign_test.go @@ -149,7 +149,7 @@ func TestSignVerifyRoundTrip(t *testing.T) { Version: "0.3.0", Servers: 1, Templates: 1, Ext: &Extended{ Schema: 1, InstallID: testID, - Env: Env{K8s: "1.31", Distro: "k3s", Arch: []string{"amd64"}, Nodes: "1"}, + Env: Env{K8s: []string{"1.31"}, Distro: []string{"k3s"}, Arch: []string{"amd64"}, Nodes: "1"}, Features: Features{Clusters: "1", DB: "sqlite", Language: "en"}, Key: PublicKeyString(priv), SentAt: time.Date(2026, 10, 6, 9, 12, 44, 0, time.UTC), diff --git a/telemetryschema/specs.md b/telemetryschema/specs.md index abdebf81b..bb76a9eeb 100644 --- a/telemetryschema/specs.md +++ b/telemetryschema/specs.md @@ -53,7 +53,7 @@ The wire contract is [`specs/022-default-telemetry-dashboard/contracts/report-sc |---|---| | `Report` | `Version string`, `Servers int`, `Templates int`, `Ext *Extended` (nil on a basic-only report) | | `Extended` | `Schema int`, `InstallID string`, `Env Env`, `Games Games`, `Features Features`, `Key string`, `SentAt time.Time` | -| `Env` | `K8s string`, `Distro string`, `Arch []string`, `Nodes string` | +| `Env` | `K8s []string`, `Distro []string`, `Arch []string`, `Nodes string`, `Deployment string` (schema 2; `other` on a schema 1 report) | | `Games` | `Official map[string]int`, `Custom int` | | `Features` | `WakeOnConnect`, `Capture`, `Backups`, `SSO`, `AuditForwarding bool`; `Tunnels []string`; `Clusters`, `DB`, `Language string` | @@ -73,6 +73,8 @@ JSON tags are the camelCase names in the contract (`installId`, `wakeOnConnect`, `SanitizeEnum(set, v)` returns `v` when it is in `set` and `other` otherwise. `K8sMinorRE` is `^1\.[0-9]{1,3}$`; any other `env.k8s` value becomes `other`. `VersionRE` is `^[A-Za-z0-9][A-Za-z0-9._+-]{0,31}$`. +`Deployments` is `cluster`, `standalone` (FR-045). + ### Bands | Function | Input to band | @@ -86,13 +88,14 @@ JSON tags are the camelCase names in the contract (`installId`, `wakeOnConnect`, `Decode(body) (Report, DecodeInfo, error)` accepts exactly one JSON object and nothing after it except whitespace. - **Rejected** (the error wraps `ErrInvalidPayload`), at every nesting level: unknown keys, duplicate keys, `null` values, wrongly typed values, keys with the wrong case, and missing required keys. `version`, `servers` and `templates` are required; `ext` is optional; every `ext` field is required. -- **Range rules:** `servers`, `templates` and `games.custom` are at least 0; `games.official` counts are at least 1; `env.arch` is non-empty; `ext.schema` is 1; `ext.key` is unpadded base64url of exactly 32 bytes; `ext.sentAt` parses as RFC 3339 and is returned in UTC. -- **Folded, not rejected:** out-of-set enumeration values become `other`. `env.arch` and `features.tunnels` are returned sorted, de-duplicated after folding, and never nil. +- **Range rules:** `servers`, `templates` and `games.custom` are at least 0; `games.official` counts are at least 1; `env.arch`, `env.k8s` and `env.distro` are non-empty; `ext.schema` is 1 or 2; `ext.key` is unpadded base64url of exactly 32 bytes; `ext.sentAt` parses as RFC 3339 and is returned in UTC. +- **Folded, not rejected:** out-of-set enumeration values become `other`. `env.arch`, `env.k8s`, `env.distro` and `features.tunnels` are returned sorted, de-duplicated after folding, and never nil. - **Catalog:** `games.official` keys outside the embedded catalog add their count to `games.custom`; the returned `Official` map holds catalog names only. -- **Schema gate:** `ext.schema` above 1 returns `ErrUnsupportedSchema`, checked before the other `ext` rules. It wraps `ErrInvalidPayload`, so one `errors.Is` check covers both. +- **Schema gate:** `ext.schema` above 2 returns `ErrUnsupportedSchema`, checked before the other `ext` rules. It wraps `ErrInvalidPayload`, so one `errors.Is` check covers both. +- **Schema 1 and 2:** schema 2 requires `env.deployment` and sends `env.k8s` and `env.distro` as arrays (OD-13, OD-15). Schema 1 refuses `env.deployment` as an unknown key and sends `env.k8s` and `env.distro` as strings; they decode as one-value sets, and `Deployment` is `other` (OD-14). - **Install ID:** when `ext.installId` is not a lowercase UUIDv4, the rest of `ext` is validated, then `DecodeInfo.ExtDropped` is set and `Ext` is nil. The basic report stays valid. -`Encode(Report)` is deterministic: struct field order is fixed, map keys and arrays are sorted, nil arrays and maps are written as `[]` and `{}`, and `sentAt` is written in UTC at second precision. `Decode(Encode(r))` returns `r` for a valid report. +`Encode(Report)` is deterministic: struct field order is fixed, map keys and arrays are sorted, nil arrays and maps are written as `[]` and `{}`, and `sentAt` is written in UTC at second precision. `Decode(Encode(r))` returns `r` for a valid schema 2 report. Schema 1 writes `env.k8s` and `env.distro` as single values and no deployment. ### Catalog diff --git a/test/e2e/telemetry_client_test.go b/test/e2e/telemetry_client_test.go index 71e3908af..06fe38869 100644 --- a/test/e2e/telemetry_client_test.go +++ b/test/e2e/telemetry_client_test.go @@ -76,7 +76,7 @@ func (c *telemetryTestClient) report(t *testing.T, installID, version string, se Ext: &telemetryschema.Extended{ Schema: telemetryschema.SchemaVersion, InstallID: installID, - Env: telemetryschema.Env{K8s: "1.31", Distro: "other", Arch: []string{"amd64"}, Nodes: "1"}, + Env: telemetryschema.Env{K8s: []string{"1.31"}, Distro: []string{"other"}, Arch: []string{"amd64"}, Nodes: "1", Deployment: "cluster"}, Games: telemetryschema.Games{Official: map[string]int{}, Custom: servers}, Features: telemetryschema.Features{Tunnels: []string{}, Clusters: "1", DB: "sqlite", Language: "en"}, Key: telemetryschema.PublicKeyString(priv), diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index e9c655d97..583d613f5 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -312,7 +312,7 @@ export interface TelemetryStatus { export interface TelemetryPreviewExt { schema: number; installId: string; - env: { k8s: string; distro: string; arch: string[]; nodes: string }; + env: { k8s: string[]; distro: string[]; arch: string[]; nodes: string; deployment: string }; games: { official: Record; custom: number }; features: { wakeOnConnect: boolean; diff --git a/web/src/routes/AdminSettings_telemetry.test.tsx b/web/src/routes/AdminSettings_telemetry.test.tsx index c27efef6e..1fedfeaa4 100644 --- a/web/src/routes/AdminSettings_telemetry.test.tsx +++ b/web/src/routes/AdminSettings_telemetry.test.tsx @@ -25,9 +25,9 @@ const PREVIEW: NonNullable = { servers: 3, templates: 7, ext: { - schema: 1, + schema: 2, installId: ID, - env: { k8s: "1.31", distro: "k3s", arch: ["amd64"], nodes: "1" }, + env: { k8s: ["1.31"], distro: ["k3s"], arch: ["amd64"], nodes: "1", deployment: "cluster" }, games: { official: { "minecraft-java": 2, terraria: 1 }, custom: 0 }, features: { wakeOnConnect: true, tunnels: ["playit"], capture: false, backups: true, diff --git a/web/src/test/browser-msw.ts b/web/src/test/browser-msw.ts index fd36b3b8e..c91067796 100644 --- a/web/src/test/browser-msw.ts +++ b/web/src/test/browser-msw.ts @@ -69,13 +69,14 @@ function getHandlerSet(): Parameters[0][] { servers: 3, templates: 8, ext: { - schema: 1, + schema: 2, installId: telemetryInstallId, env: { - k8s: "v1.28.0", - distro: "ubuntu", + k8s: ["v1.28.0"], + distro: ["ubuntu"], arch: ["amd64"], nodes: "1", + deployment: "cluster", }, games: { official: { minecraft: 2, terraria: 1 },