From 6559f54adbff0a74a69d9512b645e9775e3e5967 Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 07:28:38 +0000 Subject: [PATCH 1/9] fixup aircraft type query for old postgres --- core/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/api.go b/core/api.go index 9be578d..c62d267 100644 --- a/core/api.go +++ b/core/api.go @@ -701,7 +701,7 @@ func (s *APIServer) getTopAircraftTypes(c *gin.Context, period string, flightora SELECT t, Count(t) as count FROM ` + innerQuery + ` GROUP BY t ORDER BY count DESC - ) top_15 + ) AS top_15 ORDER BY count DESC LIMIT 15` rows, err := s.pg.db.Query(context.Background(), query) From b938cbecea12f7f14e585da7def20c090321c3c1 Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 15:19:20 +0000 Subject: [PATCH 2/9] fix timezone issue in 'seen today' statistics the previous code uses current_date but that's the date in UTC, thus it can differ from current_date in another timezone replacing CURRENT_DATE with DATE(now() at time zone TZ) would probably be the most direct fix but this fix will be faster once an index on first_seen is added using the DATE() function on a column won't gain this speedup --- core/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/api.go b/core/api.go index c62d267..07eae3e 100644 --- a/core/api.go +++ b/core/api.go @@ -152,7 +152,7 @@ func (s *APIServer) getFlightsSeenMetrics(c *gin.Context) { // Today's flights count var todayFlights int err = s.pg.db.QueryRow(context.Background(), - "SELECT COUNT(*) FROM aircraft_data WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayFlights) + "SELECT COUNT(*) FROM aircraft_data WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayFlights) if err == nil { stats["today_flights"] = todayFlights } @@ -184,7 +184,7 @@ func (s *APIServer) getAircraftSeenMetrics(c *gin.Context) { // Today's aircraft count var todayAircraft int err = s.pg.db.QueryRow(context.Background(), - "SELECT COUNT(DISTINCT hex) FROM aircraft_data WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayAircraft) + "SELECT COUNT(DISTINCT hex) FROM aircraft_data WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayAircraft) if err == nil { stats["today_aircraft"] = todayAircraft } @@ -262,7 +262,7 @@ func (s *APIServer) getInterestingMetrics(c *gin.Context) { // Today's interesting aircraft count var todayInterestingCount int err = s.pg.db.QueryRow(context.Background(), - "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayInterestingCount) + "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayInterestingCount) if err == nil { stats["today_interesting"] = todayInterestingCount } From d67591810963b735905961a1df01881f366fa071 Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 15:27:40 +0000 Subject: [PATCH 3/9] getInteresting: add error printing and fixup wrong column --- core/api.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/api.go b/core/api.go index 07eae3e..bb6fe45 100644 --- a/core/api.go +++ b/core/api.go @@ -257,22 +257,28 @@ func (s *APIServer) getInterestingMetrics(c *gin.Context) { err := s.pg.db.QueryRow(context.Background(), "SELECT COUNT(*) FROM interesting_aircraft_seen").Scan(&interestingCount) if err == nil { stats["total_interesting"] = interestingCount + } else { + log.Error().Err(err).Msg("Error getting totalInterestingCount") } // Today's interesting aircraft count var todayInterestingCount int err = s.pg.db.QueryRow(context.Background(), - "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayInterestingCount) + "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayInterestingCount) if err == nil { stats["today_interesting"] = todayInterestingCount + } else { + log.Error().Err(err).Msg("Error getting todayInterestingCount") } // Past hour interesting aircraft count var hourInterestingCount int err = s.pg.db.QueryRow(context.Background(), - "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE first_seen >= NOW() - INTERVAL '1 hour'").Scan(&hourInterestingCount) + "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE seen >= NOW() - INTERVAL '1 hour'").Scan(&hourInterestingCount) if err == nil { stats["hour_interesting"] = hourInterestingCount + } else { + log.Error().Err(err).Msg("Error getting hourInterestingCount") } c.JSON(http.StatusOK, stats) From 916c82d9762517fefe797b4608c45ab5dc66c479 Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 15:36:57 +0000 Subject: [PATCH 4/9] getTopAircraftTypes: make query work better with first_seen index --- core/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/api.go b/core/api.go index bb6fe45..83af59c 100644 --- a/core/api.go +++ b/core/api.go @@ -679,11 +679,11 @@ func (s *APIServer) getTopAircraftTypes(c *gin.Context, period string, flightora switch period { case "year": - timeFilter = `age(now(), first_seen) <= INTERVAL '1 year' AND` + timeFilter = `first_seen >= now() - INTERVAL '1 year' AND` case "month": - timeFilter = `age(now(), first_seen) <= INTERVAL '1 month' AND` + timeFilter = `first_seen >= now() - INTERVAL '1 month' AND` case "day": - timeFilter = `age(now(), first_seen) <= INTERVAL '1 day' AND` + timeFilter = `first_seen >= now() - INTERVAL '1 day' AND` default: timeFilter = "" } From 39738c4848c9951076436aa05b0f9b186aed4caf Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 15:47:47 +0000 Subject: [PATCH 5/9] readme: add notes on how to get into psql / reset password --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 6cf0327..d323a67 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,20 @@ And the following to `compose.yml` under the `skystats` service: **⚠️ The format of the csv must match the format of combined plane data + image file from plane-alert-db** +### Reset DB password + +Get a psql console: + +``` +docker exec -it skystats-db psql --dbname=skystats_db --username=skystats-user +``` + +Set new password: + +``` +alter user "skystats-user" password 'NEW_PASS'; +``` +
## Screenshots From f725f79412b965f916f1c3585a5a050e3fef49ae Mon Sep 17 00:00:00 2001 From: Matthias Wirth Date: Sat, 25 Oct 2025 15:50:05 +0000 Subject: [PATCH 6/9] add first_seen index to speed up various queries --- migrations/000007_add_first_seen_index.down.sql | 2 ++ migrations/000007_add_first_seen_index.up.sql | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 migrations/000007_add_first_seen_index.down.sql create mode 100644 migrations/000007_add_first_seen_index.up.sql diff --git a/migrations/000007_add_first_seen_index.down.sql b/migrations/000007_add_first_seen_index.down.sql new file mode 100644 index 0000000..0381384 --- /dev/null +++ b/migrations/000007_add_first_seen_index.down.sql @@ -0,0 +1,2 @@ +-- remove first_seen index +DROP INDEX IF EXISTS idx_aircraft_data_first_seen; diff --git a/migrations/000007_add_first_seen_index.up.sql b/migrations/000007_add_first_seen_index.up.sql new file mode 100644 index 0000000..0324cd3 --- /dev/null +++ b/migrations/000007_add_first_seen_index.up.sql @@ -0,0 +1,2 @@ +-- add first_seen index to speed up various queries +CREATE INDEX IF NOT EXISTS idx_aircraft_data_first_seen ON aircraft_data (first_seen DESC); From fb8011b0ab21d157d3fd29d06f775ef8fe6e9c7a Mon Sep 17 00:00:00 2001 From: Tom Carman Date: Sun, 26 Oct 2025 21:17:29 +0000 Subject: [PATCH 7/9] feat: add version to startup logs --- core/core.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/core.go b/core/core.go index 9f44eaa..6fb780b 100644 --- a/core/core.go +++ b/core/core.go @@ -112,7 +112,11 @@ func main() { if banner, err := os.ReadFile("../docs/logo/skystats_ascii.txt"); err == nil { log.Info().Msg("\n" + string(banner)) } - log.Info().Msg("Welcome to Skystats!") + if version == "dev" { + log.Info().Msgf("Welcome to Skystats! (build: %s • %s)", version, commit) + } else { + log.Info().Msgf("Welcome to Skystats %s!", version) + } defer func() { log.Info().Msg("Closing database connection") From c5676f793a7413585bc9d66ae74653d0fb24f902 Mon Sep 17 00:00:00 2001 From: Tom Carman Date: Mon, 4 May 2026 21:14:30 +0100 Subject: [PATCH 8/9] fix: api performance improvements * add index on last_seen / last_seen_distance * cache total aircraft/flights seen, refresh daily (total "aircraft" seen uniqueness was causing very slow queries as db grows. Also applied caching to total "flights" - even though less expensive as not unique) * stop interesting / routes / record holders from polling - not needed - data just fetched on tab load / refresh. --- core/api.go | 63 +++++++++++-- core/cached-stats.go | 90 +++++++++++++++++++ core/core.go | 6 +- migrations/000007_add_cached_stats.down.sql | 1 + migrations/000007_add_cached_stats.up.sql | 12 +++ migrations/000008_add_above_index.down.sql | 1 + migrations/000008_add_above_index.up.sql | 2 + web/src/components/InterestingAircraft.svelte | 11 +-- web/src/components/MetricAircraftSeen.svelte | 33 ++++--- web/src/components/MetricFlightsSeen.svelte | 32 +++++-- web/src/components/MotionStats.svelte | 11 +-- web/src/components/RouteTopAirlines.svelte | 10 +-- .../RouteTopAirportsDomestic.svelte | 10 +-- .../RouteTopAirportsInternational.svelte | 10 +-- .../RouteTopCountriesDestination.svelte | 10 +-- .../components/RouteTopCountriesOrigin.svelte | 10 +-- web/src/components/RouteTopRoutes.svelte | 10 +-- 17 files changed, 220 insertions(+), 102 deletions(-) create mode 100644 core/cached-stats.go create mode 100644 migrations/000007_add_cached_stats.down.sql create mode 100644 migrations/000007_add_cached_stats.up.sql create mode 100644 migrations/000008_add_above_index.down.sql create mode 100644 migrations/000008_add_above_index.up.sql diff --git a/core/api.go b/core/api.go index 9be578d..631ce38 100644 --- a/core/api.go +++ b/core/api.go @@ -14,9 +14,10 @@ import ( ) type APIServer struct { - pg *postgres - port string - settings *SettingsService + pg *postgres + port string + settings *SettingsService + cachedStats *CachedStatsService } func NewAPIServer(pg *postgres) *APIServer { @@ -25,9 +26,10 @@ func NewAPIServer(pg *postgres) *APIServer { port = "8080" } return &APIServer{ - pg: pg, - port: port, - settings: NewSettingsService(pg), + pg: pg, + port: port, + settings: NewSettingsService(pg), + cachedStats: NewCachedStatsService(pg), } } @@ -79,8 +81,8 @@ func (s *APIServer) Start() { { stats.GET("/above", s.getAboveStats) - stats.GET("/seen/flights", s.getFlightsSeenMetrics) - stats.GET("/seen/aircraft", s.getAircraftSeenMetrics) + stats.GET("/seen/recent", s.getRecentSeenMetrics) + stats.GET("/seen/totals", s.getTotalSeenMetrics) stats.GET("/routes/metrics", s.getRouteMetrics) stats.GET("/routes/airlines", s.getTopAirlines) @@ -119,6 +121,9 @@ func (s *APIServer) Start() { stats.GET("/charts/aircraft/month", func(c *gin.Context) { s.getChartAircraftOverTime(c, "month") }) stats.GET("/charts/aircraft/day", func(c *gin.Context) { s.getChartAircraftOverTime(c, "day") }) + // Depracacted + stats.GET("/seen/flights", s.getFlightsSeenMetrics) + stats.GET("/seen/aircraft", s.getAircraftSeenMetrics) } settings := api.Group("/settings") @@ -1461,3 +1466,45 @@ func (s *APIServer) updateSettings(c *gin.Context) { } c.JSON(http.StatusOK, settings) } + +func (s *APIServer) getRecentSeenMetrics(c *gin.Context) { + stats := gin.H{} + tz := s.getTimezone(c) + + query := ` + SELECT + COUNT(*) FILTER (WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE) as today_flights, + COUNT(*) FILTER (WHERE first_seen >= NOW() - INTERVAL '1 hour') as hour_flights, + COUNT(DISTINCT CASE WHEN DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE THEN hex END) as today_aircraft, + COUNT(DISTINCT CASE WHEN first_seen >= NOW() - INTERVAL '1 hour' THEN hex END) as hour_aircraft + FROM aircraft_data + WHERE first_seen >= CURRENT_DATE AT TIME ZONE $1 - INTERVAL '1 day' + ` + + var todayFlights, hourFlights, todayAircraft, hourAircraft int + err := s.pg.db.QueryRow(context.Background(), query, tz).Scan(&todayFlights, &hourFlights, &todayAircraft, &hourAircraft) + if err == nil { + stats["today_flights"] = todayFlights + stats["hour_flights"] = hourFlights + stats["today_aircraft"] = todayAircraft + stats["hour_aircraft"] = hourAircraft + } + + c.JSON(http.StatusOK, stats) +} + +func (s *APIServer) getTotalSeenMetrics(c *gin.Context) { + stats := gin.H{} + + totalFlights, err := s.cachedStats.GetCachedStat("total_flights") + if err == nil { + stats["total_flights"] = totalFlights + } + + totalAircraft, err := s.cachedStats.GetCachedStat("total_aircraft") + if err == nil { + stats["total_aircraft"] = totalAircraft + } + + c.JSON(http.StatusOK, stats) +} diff --git a/core/cached-stats.go b/core/cached-stats.go new file mode 100644 index 0000000..8731849 --- /dev/null +++ b/core/cached-stats.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "time" + + "github.com/rs/zerolog/log" +) + +type CachedStatsService struct { + pg *postgres +} + +func NewCachedStatsService(pg *postgres) *CachedStatsService { + return &CachedStatsService{pg: pg} +} + +func (s *CachedStatsService) GetCachedStat(key string) (int, error) { + var value int + err := s.pg.db.QueryRow(context.Background(), + "SELECT stat_value FROM cached_stats WHERE stat_key = $1", key).Scan(&value) + return value, err +} + +func (s *CachedStatsService) UpdateCachedStat(key string, value int) error { + _, err := s.pg.db.Exec(context.Background(), + `INSERT INTO cached_stats (stat_key, stat_value, last_updated) + VALUES ($1, $2, NOW()) + ON CONFLICT (stat_key) + DO UPDATE SET stat_value = $2, last_updated = NOW()`, + key, value) + return err +} + +func (s *CachedStatsService) RefreshAllTotals() error { + log.Info().Msg("Refreshing cached all-time statistics...") + + // Update total flights + var totalFlights int + err := s.pg.db.QueryRow(context.Background(), + "SELECT COUNT(*) FROM aircraft_data").Scan(&totalFlights) + if err != nil { + log.Error().Err(err).Msg("Failed to count total flights") + return err + } + err = s.UpdateCachedStat("total_flights", totalFlights) + if err != nil { + log.Error().Err(err).Msg("Failed to update cached total_flights") + return err + } + + // Update total aircraft + var totalAircraft int + err = s.pg.db.QueryRow(context.Background(), + "SELECT COUNT(DISTINCT hex) FROM aircraft_data").Scan(&totalAircraft) + if err != nil { + log.Error().Err(err).Msg("Failed to count total aircraft") + return err + } + err = s.UpdateCachedStat("total_aircraft", totalAircraft) + if err != nil { + log.Error().Err(err).Msg("Failed to update cached total_aircraft") + return err + } + + log.Info(). + Int("total_flights", totalFlights). + Int("total_aircraft", totalAircraft). + Msg("Successfully refreshed cached statistics") + + return nil +} + +func (s *CachedStatsService) StartPeriodicRefresh(interval time.Duration) { + go func() { + // Initial refresh on startup + if err := s.RefreshAllTotals(); err != nil { + log.Error().Err(err).Msg("Initial cache refresh failed") + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for range ticker.C { + if err := s.RefreshAllTotals(); err != nil { + log.Error().Err(err).Msg("Periodic cache refresh failed") + } + } + }() +} diff --git a/core/core.go b/core/core.go index 9f44eaa..3e08c1e 100644 --- a/core/core.go +++ b/core/core.go @@ -93,10 +93,14 @@ func main() { os.Exit(1) } + apiServer := NewAPIServer(pg) + + log.Info().Msg("Starting cached statistics refresh service") + apiServer.cachedStats.StartPeriodicRefresh(24 * time.Hour) + // Start API server in a separate goroutine log.Info().Msg("Starting API server") go func() { - apiServer := NewAPIServer(pg) apiServer.Start() }() diff --git a/migrations/000007_add_cached_stats.down.sql b/migrations/000007_add_cached_stats.down.sql new file mode 100644 index 0000000..bb47cd2 --- /dev/null +++ b/migrations/000007_add_cached_stats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cached_stats; diff --git a/migrations/000007_add_cached_stats.up.sql b/migrations/000007_add_cached_stats.up.sql new file mode 100644 index 0000000..b346ecc --- /dev/null +++ b/migrations/000007_add_cached_stats.up.sql @@ -0,0 +1,12 @@ +-- Create table for cached statistics +CREATE TABLE cached_stats ( + stat_key VARCHAR(50) PRIMARY KEY, + stat_value BIGINT NOT NULL, + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Insert initial values for all-time totals +INSERT INTO cached_stats (stat_key, stat_value, last_updated) +VALUES + ('total_flights', (SELECT COUNT(*) FROM aircraft_data), NOW()), + ('total_aircraft', (SELECT COUNT(DISTINCT hex) FROM aircraft_data), NOW()); diff --git a/migrations/000008_add_above_index.down.sql b/migrations/000008_add_above_index.down.sql new file mode 100644 index 0000000..c1b8bb7 --- /dev/null +++ b/migrations/000008_add_above_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_aircraft_data_last_seen_distance; diff --git a/migrations/000008_add_above_index.up.sql b/migrations/000008_add_above_index.up.sql new file mode 100644 index 0000000..e4e4acb --- /dev/null +++ b/migrations/000008_add_above_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX idx_aircraft_data_last_seen_distance +ON aircraft_data (last_seen DESC, last_seen_distance); diff --git a/web/src/components/InterestingAircraft.svelte b/web/src/components/InterestingAircraft.svelte index edffa44..039eb14 100644 --- a/web/src/components/InterestingAircraft.svelte +++ b/web/src/components/InterestingAircraft.svelte @@ -1,5 +1,5 @@ diff --git a/web/src/components/MetricFlightsSeen.svelte b/web/src/components/MetricFlightsSeen.svelte index a116dbf..e8d0ebe 100644 --- a/web/src/components/MetricFlightsSeen.svelte +++ b/web/src/components/MetricFlightsSeen.svelte @@ -5,13 +5,12 @@ import SkeletonMetrics from './SkeletonMetrics.svelte'; let data = {}; - let endpoint = 'api/stats/seen/flights' + let endpoint = 'api/stats/seen/recent' let loading = true; let error = null; - let interval = null; - - async function fetchData() { + let recentInterval = null; + async function fetchRecentData() { try { const tz = Intl.DateTimeFormat().resolvedOptions().timeZone; const response = await fetch(`${endpoint}?tz=${encodeURIComponent(tz)}`); @@ -19,7 +18,7 @@ throw new Error(`${response.status}`); } const result = await response.json(); - data = result; + data = { ...data, ...result }; error = null; } catch (err) { error = err.message; @@ -28,14 +27,29 @@ } } + async function fetchTotalData() { + try { + const response = await fetch(`api/stats/seen/totals`); + if (!response.ok) { + throw new Error(`${response.status}`); + } + const result = await response.json(); + data = { ...data, ...result }; + error = null; + } catch (err) { + error = err.message; + } + } + onMount(() => { - fetchData(); - interval = setInterval(fetchData, 2000); + fetchRecentData(); + fetchTotalData(); + recentInterval = setInterval(fetchRecentData, 2000); }) onDestroy(() => { - if (interval) { - clearInterval(interval); + if (recentInterval) { + clearInterval(recentInterval); } }); diff --git a/web/src/components/MotionStats.svelte b/web/src/components/MotionStats.svelte index d081c3d..d0dbbd6 100644 --- a/web/src/components/MotionStats.svelte +++ b/web/src/components/MotionStats.svelte @@ -1,6 +1,6 @@