Show node capacity, location and recent UPS in Fizruk status

src/Fizruk/Config.fs

@@ -18,12 +18,17 @@ type PlayersConfig =
       /// Env var holding the RCON password. Set only when Type = Rcon.
       PasswordEnv: string option }
 
+type FactorioUpsConfig =
+    { PrometheusUrl: Uri
+      MetricSelector: string }
+
 /// Extra game-specific facts appended to a running game's /status reply. Each case
 /// has its own fetcher in StatusDetails; add a case per new kind of detail.
 [<RequireQualifiedAccess>]
 type StatusDetail =
     /// Current research and its progress, read over the game's RCON endpoint.
     | FactorioResearch
+    | FactorioUps of FactorioUpsConfig
 
 /// The shared Envoy Gateway a game's ListenerSet attaches to.
 type GatewayConfig = { Name: string; Namespace: string }
@@ -193,6 +198,17 @@ module Config =
                 if players.Type <> ProbeType.Rcon then
                     raise (ConfigError $"{ctx}: details type 'factorio-research' requires players.type=rcon")
                 StatusDetail.FactorioResearch
+            | "factorio-ups" ->
+                let url = requireString entry ctx "prometheusUrl"
+                let endpoint =
+                    match Uri.TryCreate(url, UriKind.Absolute) with
+                    | true, uri when (uri.Scheme = "http" || uri.Scheme = "https")
+                                     && uri.UserInfo = "" && uri.Query = "" && uri.Fragment = "" -> uri
+                    | _ -> raise (ConfigError $"{ctx}: prometheusUrl must be an HTTP(S) base URL without credentials, query or fragment")
+                let selector = requireString entry ctx "metricSelector"
+                if String.IsNullOrWhiteSpace selector then
+                    raise (ConfigError $"{ctx}: metricSelector must not be empty")
+                StatusDetail.FactorioUps { PrometheusUrl = endpoint; MetricSelector = selector }
             | other -> raise (ConfigError $"{ctx}: unknown details type '{other}'")
         match el with
         | None -> []

src/Fizruk/Core.fs

@@ -2,6 +2,7 @@ namespace Fizruk
 
 open System
 open System.Collections.Concurrent
+open System.Globalization
 open System.Threading
 open System.Threading.Tasks
 open Microsoft.Extensions.Logging
@@ -20,7 +21,12 @@ module private StatusText =
                 match n.CreationTimestamp with
                 | Some ts -> $"{int (now - ts).TotalMinutes}m"
                 | None -> "unknown"
-            $"Node: {n.Name} ({readyText}, age {ageText})"
+            let cpu = n.CpuCount |> Option.map (fun count -> count.ToString("0.##", CultureInfo.InvariantCulture) + " vCPU")
+            let region = n.Region |> Option.map (fun region -> $"region {region}")
+            let zone = n.Zone |> Option.map (fun zone -> $"zone {zone}")
+            let facts = [ cpu; region; zone ] |> List.choose id
+            let suffix = if facts.IsEmpty then "" else "; " + String.concat ", " facts
+            $"Node: {n.Name} ({readyText}, age {ageText}{suffix})"
 
     let formatPlayersLine (players: PlayersProbeResult) =
         match players with
@@ -170,7 +176,7 @@ type GameCore(config: FizrukConfig, k8s: IK8sGateway, notifier: INotifier, time:
             let! listenerLines = this.GetListenerLines game
             let! detailLines =
                 match pod with
-                | Some p when p.Ready -> StatusDetails.fetchAll game
+                | Some p when desired > 0 && p.Ready -> StatusDetails.fetchAll game
                 | _ -> Task.FromResult []
             return StatusText.formatStatus game desired pod node players detailLines listenerLines (time.GetUtcNow())
         }

src/Fizruk/Domain.fs

@@ -14,7 +14,10 @@ type PodInfo =
 type NodeInfo =
     { Name: string
       Ready: bool
-      CreationTimestamp: DateTimeOffset option }
+      CreationTimestamp: DateTimeOffset option
+      CpuCount: decimal option
+      Region: string option
+      Zone: string option }
 
 /// What a player probe found: named players (RCON) or just a headcount (RakNet).
 [<RequireQualifiedAccess>]

src/Fizruk/K8sGateway.fs

@@ -34,6 +34,13 @@ module K8sGateway =
         let utc = if dt.Kind = DateTimeKind.Unspecified then DateTime.SpecifyKind(dt, DateTimeKind.Utc) else dt
         DateTimeOffset utc
 
+    let tryValue (key: string) (values: IDictionary<string, 'T>) : 'T option =
+        if isNull values then None
+        else
+            match values.TryGetValue key with
+            | true, value -> Some value
+            | false, _ -> None
+
     let private tryProp (el: JsonElement) (name: string) : JsonElement option =
         match el.TryGetProperty name with
         | true, v -> Some v
@@ -130,7 +137,10 @@ type KubernetesGateway(client: Kubernetes) =
                     |> Seq.map (fun n ->
                         { Name = n.Metadata.Name
                           Ready = isNodeReady n.Status.Conditions
-                          CreationTimestamp = n.Metadata.CreationTimestamp |> Option.ofNullable |> Option.map K8sGateway.asUtcOffset })
+                          CreationTimestamp = n.Metadata.CreationTimestamp |> Option.ofNullable |> Option.map K8sGateway.asUtcOffset
+                          CpuCount = n.Status.Capacity |> K8sGateway.tryValue "cpu" |> Option.map (fun quantity -> quantity.ToDecimal())
+                          Region = n.Metadata.Labels |> K8sGateway.tryValue "topology.kubernetes.io/region"
+                          Zone = n.Metadata.Labels |> K8sGateway.tryValue "topology.kubernetes.io/zone" })
                     |> List.ofSeq
             }
 

src/Fizruk/StatusDetails.fs

@@ -2,7 +2,10 @@ namespace Fizruk
 
 open System
 open System.Globalization
+open System.Net.Http
+open System.Text.Json
 open System.Text.RegularExpressions
+open System.Threading
 open System.Threading.Tasks
 
 /// Factorio's current research over RCON. The command and parsing are pure so
@@ -66,6 +69,52 @@ module FactorioResearch =
                 return format (result |> Result.bind parse)
         }
 
+module FactorioUps =
+
+    let private http = new HttpClient()
+
+    let private parse (body: string) : float option =
+        try
+            use doc = JsonDocument.Parse body
+            let root = doc.RootElement
+            if root.GetProperty("status").GetString() <> "success" then None
+            else
+                let data = root.GetProperty "data"
+                let result = data.GetProperty "result"
+                if data.GetProperty("resultType").GetString() <> "vector" || result.GetArrayLength() <> 1 then None
+                else
+                    let sample = result[0].GetProperty "value"
+                    let value = sample[1].GetString()
+                    match Double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture) with
+                    | true, ups when Double.IsFinite ups && ups >= 0.0 -> Some ups
+                    | _ -> None
+        with
+        | :? JsonException | :? Collections.Generic.KeyNotFoundException
+        | :? InvalidOperationException | :? IndexOutOfRangeException -> None
+
+    let fetch (config: FactorioUpsConfig) (timeout: TimeSpan) : Task<string> =
+        task {
+            let! value =
+                task {
+                    try
+                        // Require a recent active measurement so a paused or disconnected
+                        // server cannot report its last known UPS as current performance.
+                        let selector = config.MetricSelector
+                        let query = $"avg(avg_over_time({selector}[5m]) and (time() - timestamp({selector}) < 30))"
+                        let url = config.PrometheusUrl.AbsoluteUri.TrimEnd('/') + "/api/v1/query?query=" + Uri.EscapeDataString query
+                        use cancellation = new CancellationTokenSource(timeout)
+                        let! body = http.GetStringAsync(url, cancellation.Token)
+                        return parse body
+                    with
+                    | :? HttpRequestException | :? OperationCanceledException -> return None
+                }
+            let result =
+                match value with
+                | Some ups -> ups.ToString("0.0", CultureInfo.InvariantCulture)
+                | None -> "unavailable"
+            return $"Avg UPS (last 5m, active): {result}."
+        }
+
 /// A running game's configured `details` lines, in config order. Fetchers render
 /// their own failures as "unknown", so one broken detail never hides /status.
 module StatusDetails =
@@ -75,6 +124,7 @@ module StatusDetails =
     let fetchLine (game: GameConfig) (detail: StatusDetail) : Task<string> =
         match detail with
         | StatusDetail.FactorioResearch -> FactorioResearch.fetch game timeout
+        | StatusDetail.FactorioUps config -> FactorioUps.fetch config timeout
 
     let fetchAll (game: GameConfig) : Task<string list> =
         task {

tests/Fizruk.Tests/ConfigTests.fs

@@ -27,6 +27,15 @@ let private minimalGame (activityRegex: string) (playersJson: string) =
     }
     """
 
+[<Theory>]
+[<InlineData("file:///tmp/metrics", "simulation_ups")>]
+[<InlineData("http://user:password@metrics.example.test", "simulation_ups")>]
+[<InlineData("http://metrics.example.test?token=example", "simulation_ups")>]
+[<InlineData("http://metrics.example.test", " ")>]
+let ``UPS detail rejects invalid endpoint or empty selector`` (url, selector) =
+    let details = $$""""details":[{"type":"factorio-ups","prometheusUrl":"{{url}}","metricSelector":"{{selector}}"}],"""
+    Assert.Throws<ConfigError>(fun () -> Config.parse (minimalGame "JOIN" details) |> ignore) |> ignore
+
 [<Fact>]
 let ``valid config parses every game and chat`` () =
     let config = sampleConfig ()

tests/Fizruk.Tests/Fizruk.Tests.fsproj

@@ -18,6 +18,7 @@
     <Compile Include="K8sGatewayTests.fs" />
     <Compile Include="ListenerSetTests.fs" />
     <Compile Include="CoreFlowTests.fs" />
+    <Compile Include="StatusFlowTests.fs" />
     <Compile Include="ListenerSetFlowTests.fs" />
     <Compile Include="FakeCallHelpers.fs" />
     <Compile Include="FizrukContainerFixture.fs" />

tests/Fizruk.Tests/StatusFlowTests.fs

@@ -0,0 +1,144 @@
+module Fizruk.Tests.StatusFlowTests
+
+open System
+open System.Threading.Tasks
+open Fizruk
+open Fizruk.Tests.Fakes
+open k8s
+open Microsoft.AspNetCore.Builder
+open Microsoft.AspNetCore.Hosting
+open Microsoft.AspNetCore.Http
+open Microsoft.Extensions.Logging
+open Microsoft.Extensions.Logging.Abstractions
+open Xunit
+
+let private withApi (handler: HttpContext -> Task) (run: string -> Task<unit>) =
+    task {
+        let builder = WebApplication.CreateBuilder()
+        builder.Logging.ClearProviders() |> ignore
+        builder.WebHost.UseUrls("http://127.0.0.1:0") |> ignore
+        use app = builder.Build()
+        app.Run(RequestDelegate handler)
+        do! app.StartAsync()
+        do! run (Seq.exactlyOne app.Urls)
+        do! app.StopAsync()
+    }
+
+let private config (url: string) =
+    Config.parse $$$$"""
+    {
+      "games": { "demo": {
+        "displayName": "Demo", "namespace": "demo", "deployment": "demo",
+        "podSelector": "app=demo", "container": "server", "nodeLabelSelector": "role=game",
+        "address": "game.example.test:1234", "activityRegex": "JOIN",
+        "idleGraceMinutes": 5, "idleWindowMinutes": 10, "startTimeoutMinutes": 1,
+        "details": [{ "type": "factorio-ups", "prometheusUrl": "{{{{url}}}}/metrics",
+                      "metricSelector": "simulation_ups{server=\"demo\"}" }]
+      } }, "chats": { "-1": ["demo"] }
+    }
+    """
+
+let private node (metadata: string) (status: string) =
+    $$$$"""{"metadata": {"name":"game-node", {{{{metadata}}}} }, "status": { {{{{status}}}} }}"""
+
+let private fullNode =
+    node
+        """"labels":{"topology.kubernetes.io/region":"test-region","topology.kubernetes.io/zone":"test-zone-2"}"""
+        """"capacity":{"cpu":"4"},"allocatable":{"cpu":"3860m"},"conditions":[{"type":"Ready","status":"True"}]"""
+
+let private runStatus nodeJson statusCode metricsBody ready desired check =
+    let mutable metricsRequested = false
+    let handler (ctx: HttpContext) =
+        task {
+            ctx.Response.ContentType <- "application/json"
+            let body =
+                match ctx.Request.Path.Value with
+                | "/apis/apps/v1/namespaces/demo/deployments/demo" ->
+                    $$$$"""{"apiVersion":"apps/v1","kind":"Deployment","spec":{"replicas":{{{{desired}}}}}}"""
+                | "/api/v1/namespaces/demo/pods" ->
+                    let condition = if ready then "True" else "False"
+                    $$$$"""{"apiVersion":"v1","kind":"PodList","items":[{"metadata":{"name":"demo-pod"},"spec":{"nodeName":"game-node"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"{{{{condition}}}}"}]}}]}"""
+                | "/api/v1/nodes" ->
+                    $$$$"""{"apiVersion":"v1","kind":"NodeList","items":[{"metadata":{"name":"unrelated-node"},"status":{"capacity":{"cpu":"64"}}},{{{{nodeJson}}}}]}"""
+                | "/metrics/api/v1/query" ->
+                    metricsRequested <- true
+                    Assert.Equal(
+                        "avg(avg_over_time(simulation_ups{server=\"demo\"}[5m]) and (time() - timestamp(simulation_ups{server=\"demo\"}) < 30))",
+                        string ctx.Request.Query["query"])
+                    ctx.Response.StatusCode <- statusCode
+                    metricsBody
+                | path -> failwith $"Unexpected API request: {path}"
+            do! ctx.Response.WriteAsync body
+        }
+    withApi (fun ctx -> handler ctx :> Task) (fun url ->
+        task {
+            use client = new Kubernetes(KubernetesClientConfiguration(Host = url))
+            let core = GameCore(config url, KubernetesGateway client, FakeNotifier(), TimeProvider.System, TimeSpan.FromSeconds 30.0, NullLogger<GameCore>.Instance)
+            let! reply = core.Status "demo"
+            check reply metricsRequested
+        })
+
+let private result value =
+    $$$$"""{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[123,"{{{{value}}}}"]}]}}"""
+
+[<Theory>]
+[<InlineData("59.94", "59.9")>]
+[<InlineData("0", "0.0")>]
+let ``running status reports the hosting node capacity and location with measured UPS`` (value, expected) =
+    runStatus fullNode 200 (result value) true 1 (fun reply requested ->
+        Assert.True requested
+        Assert.Contains("Demo: running.", reply)
+        Assert.Contains("Node: game-node (Ready, age unknown; 4 vCPU, region test-region, zone test-zone-2)", reply)
+        Assert.Contains($"Avg UPS (last 5m, active): {expected}.", reply)
+        Assert.DoesNotContain("64 vCPU", reply)
+        Assert.Contains("Address: game.example.test:1234", reply))
+
+[<Theory>]
+[<InlineData(503, "unavailable")>]
+[<InlineData(200, "not json")>]
+[<InlineData(200, "{}")>]
+[<InlineData(200, "{\"status\":\"error\",\"error\":\"bad query\"}")>]
+[<InlineData(200, "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[]}}")>]
+[<InlineData(200, "{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{},{}]}}")>]
+let ``unavailable or invalid metrics do not hide running server status`` (code, body) =
+    runStatus fullNode code body true 1 (fun reply _ ->
+        Assert.Contains("Demo: running.", reply)
+        Assert.Contains("Avg UPS (last 5m, active): unavailable.", reply)
+        Assert.Contains("4 vCPU", reply))
+
+[<Theory>]
+[<InlineData("NaN")>]
+[<InlineData("+Inf")>]
+[<InlineData("-Inf")>]
+[<InlineData("-1")>]
+let ``invalid UPS values are unavailable rather than misleading numbers`` value =
+    runStatus fullNode 200 (result value) true 1 (fun reply _ ->
+        Assert.Contains("Avg UPS (last 5m, active): unavailable.", reply))
+
+[<Fact>]
+let ``missing node capacity and topology still allow status`` () =
+    let minimal = node """"labels":{}""" """"conditions":[]"""
+    runStatus minimal 200 (result "60") true 1 (fun reply _ ->
+        Assert.Contains("Node: game-node (NotReady, age unknown)", reply)
+        Assert.DoesNotContain("vCPU", reply)
+        Assert.Contains("Avg UPS (last 5m, active): 60.0.", reply))
+
+[<Theory>]
+[<InlineData(false, 1, "Demo: starting")>]
+[<InlineData(true, 0, "Demo: stopped.")>]
+let ``stopped and starting games do not query or display UPS`` (ready, desired, expected) =
+    runStatus fullNode 200 (result "60") ready desired (fun reply requested ->
+        Assert.False requested
+        Assert.StartsWith(expected, reply)
+        Assert.DoesNotContain("UPS", reply))
+
+[<Fact>]
+let ``metrics request timeout returns unavailable`` () =
+    withApi
+        (fun _ -> Task.Delay 500)
+        (fun url ->
+            task {
+                let metrics = { PrometheusUrl = Uri url; MetricSelector = "simulation_ups" }
+                let! line = FactorioUps.fetch metrics (TimeSpan.FromMilliseconds 50.0)
+                Assert.Equal("Avg UPS (last 5m, active): unavailable.", line)
+            })