Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions db/queries/queries.sql
Original file line number Diff line number Diff line change
Expand Up @@ -1101,14 +1101,38 @@ WHERE ($1::text = '' OR preset = $1::text)
ORDER BY preset, iata, source_type;

-- name: GetScopeStats :many
-- Count each table on its own; the old cross-join blew up to millions of rows
-- before COUNT(DISTINCT) (~10s).
-- Aggregate matching observations once, separately from node memberships to avoid
-- a cross-join. Empty IATAs keep the original global counts, including associations
-- whose observations have expired; the filtered aggregates are empty in that case.
WITH observation_counts AS (
SELECT p.scope_id,
COUNT(DISTINCT p.packet_hash) AS packet_count,
COUNT(DISTINCT os.observer_id) AS observer_count
FROM packet_observations po
JOIN packets p ON p.packet_hash = po.packet_hash
LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id
WHERE po.iata = ANY(sqlc.arg(iatas)::bpchar[]) AND p.scope_id IS NOT NULL
GROUP BY p.scope_id
), node_counts AS (
SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count
FROM nodes n
WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(sqlc.arg(iatas)::bpchar[]))
GROUP BY n.default_scope_id
)
SELECT
ts.name,
(SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count,
(SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count,
(SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count
CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0
THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id)
ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count,
CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0
THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)
ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count,
CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0
THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)
ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count
FROM transport_scopes ts
LEFT JOIN observation_counts oc ON oc.scope_id = ts.id
LEFT JOIN node_counts nc ON nc.scope_id = ts.id
ORDER BY ts.name;

-- ============================================================
Expand Down
101 changes: 101 additions & 0 deletions db/scope_stats_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// Copyright 2026 Beacon Contributors
// SPDX-License-Identifier: AGPL-3.0-or-later

package db

import (
"context"
"os"
"reflect"
"testing"
"time"

sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc"
"github.com/MeshCore-Beacon/beacon-server/internal/api"
"github.com/jackc/pgx/v5"
)

// Run on a migrated private database. All fixture tables and writes are rolled back.
func TestScopeStatsPostgres(t *testing.T) {
dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
conn, err := pgx.Connect(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer conn.Close(context.Background())
tx, err := conn.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(context.Background())
for _, table := range []string{"transport_scopes", "packets", "packet_observations", "observer_scopes", "nodes", "node_iatas"} {
if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil {
t.Fatal(err)
}
}
_, err = tx.Exec(ctx, `
INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint)
SELECT i,name,decode(repeat('00',16),'hex'),decode(lpad(to_hex(i),16,'0'),'hex')
FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name);
INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at)
SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',NOW(),NOW()
FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id);
INSERT INTO observer_scopes (observer_id,scope_id)
SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id);
INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count)
SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata,
'2026-01-01'::timestamptz+id*interval '1 second',0,1,0
FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'),
(5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata);
INSERT INTO nodes (id,public_key,node_type,default_scope_id)
SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,scope_id
FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,NULL)) v(i,scope_id);
INSERT INTO node_iatas (node_id,iata)
SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YYZ'),(5,'YYJ'),(6,'YVR'),(6,'YYJ'),(7,'YVR')) v(i,iata);
`)
if err != nil {
t.Fatal(err)
}
store := &Store{q: sqlc.New(tx)}
for _, tc := range []struct {
name string
iatas []string
a, b [3]int64 // packets, observers, nodes
}{
{"global", nil, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}},
{"empty filter", []string{}, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}},
{"YVR", []string{"YVR"}, [3]int64{2, 3, 2}, [3]int64{1, 1, 1}},
{"YYJ", []string{"YYJ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}},
{"YYZ", []string{"YYZ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 0}},
{"overlapping IATAs", []string{"YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}},
{"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}},
{"unknown IATA", []string{"ZZZ"}, [3]int64{}, [3]int64{}},
} {
t.Run(tc.name, func(t *testing.T) {
rows, err := store.GetScopeStats(ctx, tc.iatas)
if err != nil {
t.Fatal(err)
}
want := []api.ScopeStats{
{Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2]},
{Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2]},
{Name: "#unused"},
}
if !reflect.DeepEqual(rows, want) {
t.Fatalf("counts = %+v, want %+v", rows, want)
}
})
}
if _, err := tx.Exec(ctx, "DELETE FROM transport_scopes"); err != nil {
t.Fatal(err)
}
rows, err := store.GetScopeStats(ctx, nil)
if err != nil || rows == nil || len(rows) != 0 {
t.Fatalf("empty roster = %+v, %v", rows, err)
}
}
8 changes: 4 additions & 4 deletions db/sqlc/mock/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions db/sqlc/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 31 additions & 7 deletions db/sqlc/queries.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions db/stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,8 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri
return items, nil
}

func (s *Store) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) {
rows, err := s.q.GetScopeStats(ctx)
func (s *Store) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) {
rows, err := s.q.GetScopeStats(ctx, iatas)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions db/stats_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,13 +227,13 @@ func TestGetScopeStats(t *testing.T) {
mock := mockdb.NewMockQuerier(ctrl)

mock.EXPECT().
GetScopeStats(gomock.Any()).
GetScopeStats(gomock.Any(), []string{"YVR"}).
Return([]sqlc.GetScopeStatsRow{
{Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20},
}, nil)

store := &Store{q: mock}
items, err := store.GetScopeStats(context.Background())
items, err := store.GetScopeStats(context.Background(), []string{"YVR"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand Down
33 changes: 33 additions & 0 deletions docs/docs.go
Original file line number Diff line number Diff line change
Expand Up @@ -1988,13 +1988,40 @@ const docTemplate = `{
},
"/stats/scopes": {
"get": {
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.",
"produces": [
"application/json"
],
"tags": [
"Stats"
],
"summary": "Scope statistics",
"parameters": [
{
"type": "string",
"description": "Comma-separated IATA codes",
"name": "iatas",
"in": "query"
},
{
"type": "string",
"description": "Single IATA code; used when iatas is absent",
"name": "iata",
"in": "query"
},
{
"type": "integer",
"description": "Filter by region ID, expands to member IATAs",
"name": "regionId",
"in": "query"
},
{
"type": "string",
"description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs",
"name": "region",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
Expand All @@ -2005,6 +2032,12 @@ const docTemplate = `{
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
Expand Down
33 changes: 33 additions & 0 deletions docs/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -1986,13 +1986,40 @@
},
"/stats/scopes": {
"get": {
"description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.",
"produces": [
"application/json"
],
"tags": [
"Stats"
],
"summary": "Scope statistics",
"parameters": [
{
"type": "string",
"description": "Comma-separated IATA codes",
"name": "iatas",
"in": "query"
},
{
"type": "string",
"description": "Single IATA code; used when iatas is absent",
"name": "iata",
"in": "query"
},
{
"type": "integer",
"description": "Filter by region ID, expands to member IATAs",
"name": "regionId",
"in": "query"
},
{
"type": "string",
"description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs",
"name": "region",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
Expand All @@ -2003,6 +2030,12 @@
}
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/internal_api_handlers.APIError"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
Expand Down
Loading
Loading