From c5676f793a7413585bc9d66ae74653d0fb24f902 Mon Sep 17 00:00:00 2001 From: Tom Carman Date: Mon, 4 May 2026 21:14:30 +0100 Subject: [PATCH] fix: api performance improvements * add index on last_seen / last_seen_distance * cache total aircraft/flights seen, refresh daily (total "aircraft" seen uniqueness was causing very slow queries as db grows. Also applied caching to total "flights" - even though less expensive as not unique) * stop interesting / routes / record holders from polling - not needed - data just fetched on tab load / refresh. --- core/api.go | 63 +++++++++++-- core/cached-stats.go | 90 +++++++++++++++++++ core/core.go | 6 +- migrations/000007_add_cached_stats.down.sql | 1 + migrations/000007_add_cached_stats.up.sql | 12 +++ migrations/000008_add_above_index.down.sql | 1 + migrations/000008_add_above_index.up.sql | 2 + web/src/components/InterestingAircraft.svelte | 11 +-- web/src/components/MetricAircraftSeen.svelte | 33 ++++--- web/src/components/MetricFlightsSeen.svelte | 32 +++++-- web/src/components/MotionStats.svelte | 11 +-- web/src/components/RouteTopAirlines.svelte | 10 +-- .../RouteTopAirportsDomestic.svelte | 10 +-- .../RouteTopAirportsInternational.svelte | 10 +-- .../RouteTopCountriesDestination.svelte | 10 +-- .../components/RouteTopCountriesOrigin.svelte | 10 +-- web/src/components/RouteTopRoutes.svelte | 10 +-- 17 files changed, 220 insertions(+), 102 deletions(-) create mode 100644 core/cached-stats.go create mode 100644 migrations/000007_add_cached_stats.down.sql create mode 100644 migrations/000007_add_cached_stats.up.sql create mode 100644 migrations/000008_add_above_index.down.sql create mode 100644 migrations/000008_add_above_index.up.sql diff --git a/core/api.go b/core/api.go index 9be578d..631ce38 100644 --- a/core/api.go +++ b/core/api.go @@ -14,9 +14,10 @@ import ( ) type APIServer struct { - pg *postgres - port string - settings *SettingsService + pg *postgres + port string + settings *SettingsService + cachedStats *CachedStatsService } func NewAPIServer(pg *postgres) *APIServer { @@ -25,9 +26,10 @@ func NewAPIServer(pg *postgres) *APIServer { port = "8080" } return &APIServer{ - pg: pg, - port: port, - settings: NewSettingsService(pg), + pg: pg, + port: port, + settings: NewSettingsService(pg), + cachedStats: NewCachedStatsService(pg), } } @@ -79,8 +81,8 @@ func (s *APIServer) Start() { { stats.GET("/above", s.getAboveStats) - stats.GET("/seen/flights", s.getFlightsSeenMetrics) - stats.GET("/seen/aircraft", s.getAircraftSeenMetrics) + stats.GET("/seen/recent", s.getRecentSeenMetrics) + stats.GET("/seen/totals", s.getTotalSeenMetrics) stats.GET("/routes/metrics", s.getRouteMetrics) stats.GET("/routes/airlines", s.getTopAirlines) @@ -119,6 +121,9 @@ func (s *APIServer) Start() { stats.GET("/charts/aircraft/month", func(c *gin.Context) { s.getChartAircraftOverTime(c, "month") }) stats.GET("/charts/aircraft/day", func(c *gin.Context) { s.getChartAircraftOverTime(c, "day") }) + // Depracacted + stats.GET("/seen/flights", s.getFlightsSeenMetrics) + stats.GET("/seen/aircraft", s.getAircraftSeenMetrics) } settings := api.Group("/settings") @@ -1461,3 +1466,45 @@ func (s *APIServer) updateSettings(c *gin.Context) { } c.JSON(http.StatusOK, settings) } + +func (s *APIServer) getRecentSeenMetrics(c *gin.Context) { + stats := gin.H{} + tz := s.getTimezone(c) + + query := ` + SELECT + COUNT(*) FILTER (WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE) as today_flights, + COUNT(*) FILTER (WHERE first_seen >= NOW() - INTERVAL '1 hour') as hour_flights, + COUNT(DISTINCT CASE WHEN DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE THEN hex END) as today_aircraft, + COUNT(DISTINCT CASE WHEN first_seen >= NOW() - INTERVAL '1 hour' THEN hex END) as hour_aircraft + FROM aircraft_data + WHERE first_seen >= CURRENT_DATE AT TIME ZONE $1 - INTERVAL '1 day' + ` + + var todayFlights, hourFlights, todayAircraft, hourAircraft int + err := s.pg.db.QueryRow(context.Background(), query, tz).Scan(&todayFlights, &hourFlights, &todayAircraft, &hourAircraft) + if err == nil { + stats["today_flights"] = todayFlights + stats["hour_flights"] = hourFlights + stats["today_aircraft"] = todayAircraft + stats["hour_aircraft"] = hourAircraft + } + + c.JSON(http.StatusOK, stats) +} + +func (s *APIServer) getTotalSeenMetrics(c *gin.Context) { + stats := gin.H{} + + totalFlights, err := s.cachedStats.GetCachedStat("total_flights") + if err == nil { + stats["total_flights"] = totalFlights + } + + totalAircraft, err := s.cachedStats.GetCachedStat("total_aircraft") + if err == nil { + stats["total_aircraft"] = totalAircraft + } + + c.JSON(http.StatusOK, stats) +} diff --git a/core/cached-stats.go b/core/cached-stats.go new file mode 100644 index 0000000..8731849 --- /dev/null +++ b/core/cached-stats.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "time" + + "github.com/rs/zerolog/log" +) + +type CachedStatsService struct { + pg *postgres +} + +func NewCachedStatsService(pg *postgres) *CachedStatsService { + return &CachedStatsService{pg: pg} +} + +func (s *CachedStatsService) GetCachedStat(key string) (int, error) { + var value int + err := s.pg.db.QueryRow(context.Background(), + "SELECT stat_value FROM cached_stats WHERE stat_key = $1", key).Scan(&value) + return value, err +} + +func (s *CachedStatsService) UpdateCachedStat(key string, value int) error { + _, err := s.pg.db.Exec(context.Background(), + `INSERT INTO cached_stats (stat_key, stat_value, last_updated) + VALUES ($1, $2, NOW()) + ON CONFLICT (stat_key) + DO UPDATE SET stat_value = $2, last_updated = NOW()`, + key, value) + return err +} + +func (s *CachedStatsService) RefreshAllTotals() error { + log.Info().Msg("Refreshing cached all-time statistics...") + + // Update total flights + var totalFlights int + err := s.pg.db.QueryRow(context.Background(), + "SELECT COUNT(*) FROM aircraft_data").Scan(&totalFlights) + if err != nil { + log.Error().Err(err).Msg("Failed to count total flights") + return err + } + err = s.UpdateCachedStat("total_flights", totalFlights) + if err != nil { + log.Error().Err(err).Msg("Failed to update cached total_flights") + return err + } + + // Update total aircraft + var totalAircraft int + err = s.pg.db.QueryRow(context.Background(), + "SELECT COUNT(DISTINCT hex) FROM aircraft_data").Scan(&totalAircraft) + if err != nil { + log.Error().Err(err).Msg("Failed to count total aircraft") + return err + } + err = s.UpdateCachedStat("total_aircraft", totalAircraft) + if err != nil { + log.Error().Err(err).Msg("Failed to update cached total_aircraft") + return err + } + + log.Info(). + Int("total_flights", totalFlights). + Int("total_aircraft", totalAircraft). + Msg("Successfully refreshed cached statistics") + + return nil +} + +func (s *CachedStatsService) StartPeriodicRefresh(interval time.Duration) { + go func() { + // Initial refresh on startup + if err := s.RefreshAllTotals(); err != nil { + log.Error().Err(err).Msg("Initial cache refresh failed") + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for range ticker.C { + if err := s.RefreshAllTotals(); err != nil { + log.Error().Err(err).Msg("Periodic cache refresh failed") + } + } + }() +} diff --git a/core/core.go b/core/core.go index 9f44eaa..3e08c1e 100644 --- a/core/core.go +++ b/core/core.go @@ -93,10 +93,14 @@ func main() { os.Exit(1) } + apiServer := NewAPIServer(pg) + + log.Info().Msg("Starting cached statistics refresh service") + apiServer.cachedStats.StartPeriodicRefresh(24 * time.Hour) + // Start API server in a separate goroutine log.Info().Msg("Starting API server") go func() { - apiServer := NewAPIServer(pg) apiServer.Start() }() diff --git a/migrations/000007_add_cached_stats.down.sql b/migrations/000007_add_cached_stats.down.sql new file mode 100644 index 0000000..bb47cd2 --- /dev/null +++ b/migrations/000007_add_cached_stats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cached_stats; diff --git a/migrations/000007_add_cached_stats.up.sql b/migrations/000007_add_cached_stats.up.sql new file mode 100644 index 0000000..b346ecc --- /dev/null +++ b/migrations/000007_add_cached_stats.up.sql @@ -0,0 +1,12 @@ +-- Create table for cached statistics +CREATE TABLE cached_stats ( + stat_key VARCHAR(50) PRIMARY KEY, + stat_value BIGINT NOT NULL, + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Insert initial values for all-time totals +INSERT INTO cached_stats (stat_key, stat_value, last_updated) +VALUES + ('total_flights', (SELECT COUNT(*) FROM aircraft_data), NOW()), + ('total_aircraft', (SELECT COUNT(DISTINCT hex) FROM aircraft_data), NOW()); diff --git a/migrations/000008_add_above_index.down.sql b/migrations/000008_add_above_index.down.sql new file mode 100644 index 0000000..c1b8bb7 --- /dev/null +++ b/migrations/000008_add_above_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_aircraft_data_last_seen_distance; diff --git a/migrations/000008_add_above_index.up.sql b/migrations/000008_add_above_index.up.sql new file mode 100644 index 0000000..e4e4acb --- /dev/null +++ b/migrations/000008_add_above_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX idx_aircraft_data_last_seen_distance +ON aircraft_data (last_seen DESC, last_seen_distance); diff --git a/web/src/components/InterestingAircraft.svelte b/web/src/components/InterestingAircraft.svelte index edffa44..039eb14 100644 --- a/web/src/components/InterestingAircraft.svelte +++ b/web/src/components/InterestingAircraft.svelte @@ -1,5 +1,5 @@ diff --git a/web/src/components/MetricFlightsSeen.svelte b/web/src/components/MetricFlightsSeen.svelte index a116dbf..e8d0ebe 100644 --- a/web/src/components/MetricFlightsSeen.svelte +++ b/web/src/components/MetricFlightsSeen.svelte @@ -5,13 +5,12 @@ import SkeletonMetrics from './SkeletonMetrics.svelte'; let data = {}; - let endpoint = 'api/stats/seen/flights' + let endpoint = 'api/stats/seen/recent' let loading = true; let error = null; - let interval = null; - - async function fetchData() { + let recentInterval = null; + async function fetchRecentData() { try { const tz = Intl.DateTimeFormat().resolvedOptions().timeZone; const response = await fetch(`${endpoint}?tz=${encodeURIComponent(tz)}`); @@ -19,7 +18,7 @@ throw new Error(`${response.status}`); } const result = await response.json(); - data = result; + data = { ...data, ...result }; error = null; } catch (err) { error = err.message; @@ -28,14 +27,29 @@ } } + async function fetchTotalData() { + try { + const response = await fetch(`api/stats/seen/totals`); + if (!response.ok) { + throw new Error(`${response.status}`); + } + const result = await response.json(); + data = { ...data, ...result }; + error = null; + } catch (err) { + error = err.message; + } + } + onMount(() => { - fetchData(); - interval = setInterval(fetchData, 2000); + fetchRecentData(); + fetchTotalData(); + recentInterval = setInterval(fetchRecentData, 2000); }) onDestroy(() => { - if (interval) { - clearInterval(interval); + if (recentInterval) { + clearInterval(recentInterval); } }); diff --git a/web/src/components/MotionStats.svelte b/web/src/components/MotionStats.svelte index d081c3d..d0dbbd6 100644 --- a/web/src/components/MotionStats.svelte +++ b/web/src/components/MotionStats.svelte @@ -1,6 +1,6 @@