diff --git a/src/Fizruk/Config.fs b/src/Fizruk/Config.fs index 38899fa..9fe0470 100644 --- a/src/Fizruk/Config.fs +++ b/src/Fizruk/Config.fs @@ -18,12 +18,17 @@ type PlayersConfig = /// Env var holding the RCON password. Set only when Type = Rcon. PasswordEnv: string option } +type FactorioUpsConfig = + { PrometheusUrl: Uri + MetricSelector: string } + /// Extra game-specific facts appended to a running game's /status reply. Each case /// has its own fetcher in StatusDetails; add a case per new kind of detail. [] type StatusDetail = /// Current research and its progress, read over the game's RCON endpoint. | FactorioResearch + | FactorioUps of FactorioUpsConfig /// The shared Envoy Gateway a game's ListenerSet attaches to. type GatewayConfig = { Name: string; Namespace: string } @@ -193,6 +198,17 @@ module Config = if players.Type <> ProbeType.Rcon then raise (ConfigError $"{ctx}: details type 'factorio-research' requires players.type=rcon") StatusDetail.FactorioResearch + | "factorio-ups" -> + let url = requireString entry ctx "prometheusUrl" + let endpoint = + match Uri.TryCreate(url, UriKind.Absolute) with + | true, uri when (uri.Scheme = "http" || uri.Scheme = "https") + && uri.UserInfo = "" && uri.Query = "" && uri.Fragment = "" -> uri + | _ -> raise (ConfigError $"{ctx}: prometheusUrl must be an HTTP(S) base URL without credentials, query or fragment") + let selector = requireString entry ctx "metricSelector" + if String.IsNullOrWhiteSpace selector then + raise (ConfigError $"{ctx}: metricSelector must not be empty") + StatusDetail.FactorioUps { PrometheusUrl = endpoint; MetricSelector = selector } | other -> raise (ConfigError $"{ctx}: unknown details type '{other}'") match el with | None -> [] diff --git a/src/Fizruk/Core.fs b/src/Fizruk/Core.fs index 2fe349e..9bddbfe 100644 --- a/src/Fizruk/Core.fs +++ b/src/Fizruk/Core.fs @@ -2,6 +2,7 @@ namespace Fizruk open System open System.Collections.Concurrent +open System.Globalization open System.Threading open System.Threading.Tasks open Microsoft.Extensions.Logging @@ -20,7 +21,12 @@ module private StatusText = match n.CreationTimestamp with | Some ts -> $"{int (now - ts).TotalMinutes}m" | None -> "unknown" - $"Node: {n.Name} ({readyText}, age {ageText})" + let cpu = n.CpuCount |> Option.map (fun count -> count.ToString("0.##", CultureInfo.InvariantCulture) + " vCPU") + let region = n.Region |> Option.map (fun region -> $"region {region}") + let zone = n.Zone |> Option.map (fun zone -> $"zone {zone}") + let facts = [ cpu; region; zone ] |> List.choose id + let suffix = if facts.IsEmpty then "" else "; " + String.concat ", " facts + $"Node: {n.Name} ({readyText}, age {ageText}{suffix})" let formatPlayersLine (players: PlayersProbeResult) = match players with @@ -170,7 +176,7 @@ type GameCore(config: FizrukConfig, k8s: IK8sGateway, notifier: INotifier, time: let! listenerLines = this.GetListenerLines game let! detailLines = match pod with - | Some p when p.Ready -> StatusDetails.fetchAll game + | Some p when desired > 0 && p.Ready -> StatusDetails.fetchAll game | _ -> Task.FromResult [] return StatusText.formatStatus game desired pod node players detailLines listenerLines (time.GetUtcNow()) } diff --git a/src/Fizruk/Domain.fs b/src/Fizruk/Domain.fs index 7cfa815..c7b6293 100644 --- a/src/Fizruk/Domain.fs +++ b/src/Fizruk/Domain.fs @@ -14,7 +14,10 @@ type PodInfo = type NodeInfo = { Name: string Ready: bool - CreationTimestamp: DateTimeOffset option } + CreationTimestamp: DateTimeOffset option + CpuCount: decimal option + Region: string option + Zone: string option } /// What a player probe found: named players (RCON) or just a headcount (RakNet). [] diff --git a/src/Fizruk/K8sGateway.fs b/src/Fizruk/K8sGateway.fs index 6b7ebb2..c9842b7 100644 --- a/src/Fizruk/K8sGateway.fs +++ b/src/Fizruk/K8sGateway.fs @@ -34,6 +34,13 @@ module K8sGateway = let utc = if dt.Kind = DateTimeKind.Unspecified then DateTime.SpecifyKind(dt, DateTimeKind.Utc) else dt DateTimeOffset utc + let tryValue (key: string) (values: IDictionary) : 'T option = + if isNull values then None + else + match values.TryGetValue key with + | true, value -> Some value + | false, _ -> None + let private tryProp (el: JsonElement) (name: string) : JsonElement option = match el.TryGetProperty name with | true, v -> Some v @@ -130,7 +137,10 @@ type KubernetesGateway(client: Kubernetes) = |> Seq.map (fun n -> { Name = n.Metadata.Name Ready = isNodeReady n.Status.Conditions - CreationTimestamp = n.Metadata.CreationTimestamp |> Option.ofNullable |> Option.map K8sGateway.asUtcOffset }) + CreationTimestamp = n.Metadata.CreationTimestamp |> Option.ofNullable |> Option.map K8sGateway.asUtcOffset + CpuCount = n.Status.Capacity |> K8sGateway.tryValue "cpu" |> Option.map (fun quantity -> quantity.ToDecimal()) + Region = n.Metadata.Labels |> K8sGateway.tryValue "topology.kubernetes.io/region" + Zone = n.Metadata.Labels |> K8sGateway.tryValue "topology.kubernetes.io/zone" }) |> List.ofSeq } diff --git a/src/Fizruk/StatusDetails.fs b/src/Fizruk/StatusDetails.fs index b1e52a4..b783102 100644 --- a/src/Fizruk/StatusDetails.fs +++ b/src/Fizruk/StatusDetails.fs @@ -2,7 +2,10 @@ namespace Fizruk open System open System.Globalization +open System.Net.Http +open System.Text.Json open System.Text.RegularExpressions +open System.Threading open System.Threading.Tasks /// Factorio's current research over RCON. The command and parsing are pure so @@ -66,6 +69,52 @@ module FactorioResearch = return format (result |> Result.bind parse) } +module FactorioUps = + + let private http = new HttpClient() + + let private parse (body: string) : float option = + try + use doc = JsonDocument.Parse body + let root = doc.RootElement + if root.GetProperty("status").GetString() <> "success" then None + else + let data = root.GetProperty "data" + let result = data.GetProperty "result" + if data.GetProperty("resultType").GetString() <> "vector" || result.GetArrayLength() <> 1 then None + else + let sample = result[0].GetProperty "value" + let value = sample[1].GetString() + match Double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture) with + | true, ups when Double.IsFinite ups && ups >= 0.0 -> Some ups + | _ -> None + with + | :? JsonException | :? Collections.Generic.KeyNotFoundException + | :? InvalidOperationException | :? IndexOutOfRangeException -> None + + let fetch (config: FactorioUpsConfig) (timeout: TimeSpan) : Task = + task { + let! value = + task { + try + // Require a recent active measurement so a paused or disconnected + // server cannot report its last known UPS as current performance. + let selector = config.MetricSelector + let query = $"avg(avg_over_time({selector}[5m]) and (time() - timestamp({selector}) < 30))" + let url = config.PrometheusUrl.AbsoluteUri.TrimEnd('/') + "/api/v1/query?query=" + Uri.EscapeDataString query + use cancellation = new CancellationTokenSource(timeout) + let! body = http.GetStringAsync(url, cancellation.Token) + return parse body + with + | :? HttpRequestException | :? OperationCanceledException -> return None + } + let result = + match value with + | Some ups -> ups.ToString("0.0", CultureInfo.InvariantCulture) + | None -> "unavailable" + return $"Avg UPS (last 5m, active): {result}." + } + /// A running game's configured `details` lines, in config order. Fetchers render /// their own failures as "unknown", so one broken detail never hides /status. module StatusDetails = @@ -75,6 +124,7 @@ module StatusDetails = let fetchLine (game: GameConfig) (detail: StatusDetail) : Task = match detail with | StatusDetail.FactorioResearch -> FactorioResearch.fetch game timeout + | StatusDetail.FactorioUps config -> FactorioUps.fetch config timeout let fetchAll (game: GameConfig) : Task = task { diff --git a/tests/Fizruk.Tests/ConfigTests.fs b/tests/Fizruk.Tests/ConfigTests.fs index 791c585..3078095 100644 --- a/tests/Fizruk.Tests/ConfigTests.fs +++ b/tests/Fizruk.Tests/ConfigTests.fs @@ -27,6 +27,15 @@ let private minimalGame (activityRegex: string) (playersJson: string) = } """ +[] +[] +[] +[] +[] +let ``UPS detail rejects invalid endpoint or empty selector`` (url, selector) = + let details = $$""""details":[{"type":"factorio-ups","prometheusUrl":"{{url}}","metricSelector":"{{selector}}"}],""" + Assert.Throws(fun () -> Config.parse (minimalGame "JOIN" details) |> ignore) |> ignore + [] let ``valid config parses every game and chat`` () = let config = sampleConfig () diff --git a/tests/Fizruk.Tests/Fizruk.Tests.fsproj b/tests/Fizruk.Tests/Fizruk.Tests.fsproj index 50a66dd..f5d9906 100644 --- a/tests/Fizruk.Tests/Fizruk.Tests.fsproj +++ b/tests/Fizruk.Tests/Fizruk.Tests.fsproj @@ -18,6 +18,7 @@ + diff --git a/tests/Fizruk.Tests/StatusFlowTests.fs b/tests/Fizruk.Tests/StatusFlowTests.fs new file mode 100644 index 0000000..c023a3e --- /dev/null +++ b/tests/Fizruk.Tests/StatusFlowTests.fs @@ -0,0 +1,144 @@ +module Fizruk.Tests.StatusFlowTests + +open System +open System.Threading.Tasks +open Fizruk +open Fizruk.Tests.Fakes +open k8s +open Microsoft.AspNetCore.Builder +open Microsoft.AspNetCore.Hosting +open Microsoft.AspNetCore.Http +open Microsoft.Extensions.Logging +open Microsoft.Extensions.Logging.Abstractions +open Xunit + +let private withApi (handler: HttpContext -> Task) (run: string -> Task) = + task { + let builder = WebApplication.CreateBuilder() + builder.Logging.ClearProviders() |> ignore + builder.WebHost.UseUrls("http://127.0.0.1:0") |> ignore + use app = builder.Build() + app.Run(RequestDelegate handler) + do! app.StartAsync() + do! run (Seq.exactlyOne app.Urls) + do! app.StopAsync() + } + +let private config (url: string) = + Config.parse $$$$""" + { + "games": { "demo": { + "displayName": "Demo", "namespace": "demo", "deployment": "demo", + "podSelector": "app=demo", "container": "server", "nodeLabelSelector": "role=game", + "address": "game.example.test:1234", "activityRegex": "JOIN", + "idleGraceMinutes": 5, "idleWindowMinutes": 10, "startTimeoutMinutes": 1, + "details": [{ "type": "factorio-ups", "prometheusUrl": "{{{{url}}}}/metrics", + "metricSelector": "simulation_ups{server=\"demo\"}" }] + } }, "chats": { "-1": ["demo"] } + } + """ + +let private node (metadata: string) (status: string) = + $$$$"""{"metadata": {"name":"game-node", {{{{metadata}}}} }, "status": { {{{{status}}}} }}""" + +let private fullNode = + node + """"labels":{"topology.kubernetes.io/region":"test-region","topology.kubernetes.io/zone":"test-zone-2"}""" + """"capacity":{"cpu":"4"},"allocatable":{"cpu":"3860m"},"conditions":[{"type":"Ready","status":"True"}]""" + +let private runStatus nodeJson statusCode metricsBody ready desired check = + let mutable metricsRequested = false + let handler (ctx: HttpContext) = + task { + ctx.Response.ContentType <- "application/json" + let body = + match ctx.Request.Path.Value with + | "/apis/apps/v1/namespaces/demo/deployments/demo" -> + $$$$"""{"apiVersion":"apps/v1","kind":"Deployment","spec":{"replicas":{{{{desired}}}}}}""" + | "/api/v1/namespaces/demo/pods" -> + let condition = if ready then "True" else "False" + $$$$"""{"apiVersion":"v1","kind":"PodList","items":[{"metadata":{"name":"demo-pod"},"spec":{"nodeName":"game-node"},"status":{"phase":"Running","conditions":[{"type":"Ready","status":"{{{{condition}}}}"}]}}]}""" + | "/api/v1/nodes" -> + $$$$"""{"apiVersion":"v1","kind":"NodeList","items":[{"metadata":{"name":"unrelated-node"},"status":{"capacity":{"cpu":"64"}}},{{{{nodeJson}}}}]}""" + | "/metrics/api/v1/query" -> + metricsRequested <- true + Assert.Equal( + "avg(avg_over_time(simulation_ups{server=\"demo\"}[5m]) and (time() - timestamp(simulation_ups{server=\"demo\"}) < 30))", + string ctx.Request.Query["query"]) + ctx.Response.StatusCode <- statusCode + metricsBody + | path -> failwith $"Unexpected API request: {path}" + do! ctx.Response.WriteAsync body + } + withApi (fun ctx -> handler ctx :> Task) (fun url -> + task { + use client = new Kubernetes(KubernetesClientConfiguration(Host = url)) + let core = GameCore(config url, KubernetesGateway client, FakeNotifier(), TimeProvider.System, TimeSpan.FromSeconds 30.0, NullLogger.Instance) + let! reply = core.Status "demo" + check reply metricsRequested + }) + +let private result value = + $$$$"""{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[123,"{{{{value}}}}"]}]}}""" + +[] +[] +[] +let ``running status reports the hosting node capacity and location with measured UPS`` (value, expected) = + runStatus fullNode 200 (result value) true 1 (fun reply requested -> + Assert.True requested + Assert.Contains("Demo: running.", reply) + Assert.Contains("Node: game-node (Ready, age unknown; 4 vCPU, region test-region, zone test-zone-2)", reply) + Assert.Contains($"Avg UPS (last 5m, active): {expected}.", reply) + Assert.DoesNotContain("64 vCPU", reply) + Assert.Contains("Address: game.example.test:1234", reply)) + +[] +[] +[] +[] +[] +[] +[] +let ``unavailable or invalid metrics do not hide running server status`` (code, body) = + runStatus fullNode code body true 1 (fun reply _ -> + Assert.Contains("Demo: running.", reply) + Assert.Contains("Avg UPS (last 5m, active): unavailable.", reply) + Assert.Contains("4 vCPU", reply)) + +[] +[] +[] +[] +[] +let ``invalid UPS values are unavailable rather than misleading numbers`` value = + runStatus fullNode 200 (result value) true 1 (fun reply _ -> + Assert.Contains("Avg UPS (last 5m, active): unavailable.", reply)) + +[] +let ``missing node capacity and topology still allow status`` () = + let minimal = node """"labels":{}""" """"conditions":[]""" + runStatus minimal 200 (result "60") true 1 (fun reply _ -> + Assert.Contains("Node: game-node (NotReady, age unknown)", reply) + Assert.DoesNotContain("vCPU", reply) + Assert.Contains("Avg UPS (last 5m, active): 60.0.", reply)) + +[] +[] +[] +let ``stopped and starting games do not query or display UPS`` (ready, desired, expected) = + runStatus fullNode 200 (result "60") ready desired (fun reply requested -> + Assert.False requested + Assert.StartsWith(expected, reply) + Assert.DoesNotContain("UPS", reply)) + +[] +let ``metrics request timeout returns unavailable`` () = + withApi + (fun _ -> Task.Delay 500) + (fun url -> + task { + let metrics = { PrometheusUrl = Uri url; MetricSelector = "simulation_ups" } + let! line = FactorioUps.fetch metrics (TimeSpan.FromMilliseconds 50.0) + Assert.Equal("Avg UPS (last 5m, active): unavailable.", line) + })