diff --git a/README.md b/README.md
index 6cf0327..d323a67 100644
--- a/README.md
+++ b/README.md
@@ -123,6 +123,20 @@ And the following to `compose.yml` under the `skystats` service:
**⚠️ The format of the csv must match the format of combined plane data + image file from plane-alert-db**
+### Reset DB password
+
+Get a psql console:
+
+```
+docker exec -it skystats-db psql --dbname=skystats_db --username=skystats-user
+```
+
+Set new password:
+
+```
+alter user "skystats-user" password 'NEW_PASS';
+```
+
## Screenshots
diff --git a/core/api.go b/core/api.go
index 9be578d..86547f9 100644
--- a/core/api.go
+++ b/core/api.go
@@ -14,9 +14,10 @@ import (
)
type APIServer struct {
- pg *postgres
- port string
- settings *SettingsService
+ pg *postgres
+ port string
+ settings *SettingsService
+ cachedStats *CachedStatsService
}
func NewAPIServer(pg *postgres) *APIServer {
@@ -25,9 +26,10 @@ func NewAPIServer(pg *postgres) *APIServer {
port = "8080"
}
return &APIServer{
- pg: pg,
- port: port,
- settings: NewSettingsService(pg),
+ pg: pg,
+ port: port,
+ settings: NewSettingsService(pg),
+ cachedStats: NewCachedStatsService(pg),
}
}
@@ -79,8 +81,8 @@ func (s *APIServer) Start() {
{
stats.GET("/above", s.getAboveStats)
- stats.GET("/seen/flights", s.getFlightsSeenMetrics)
- stats.GET("/seen/aircraft", s.getAircraftSeenMetrics)
+ stats.GET("/seen/recent", s.getRecentSeenMetrics)
+ stats.GET("/seen/totals", s.getTotalSeenMetrics)
stats.GET("/routes/metrics", s.getRouteMetrics)
stats.GET("/routes/airlines", s.getTopAirlines)
@@ -119,6 +121,9 @@ func (s *APIServer) Start() {
stats.GET("/charts/aircraft/month", func(c *gin.Context) { s.getChartAircraftOverTime(c, "month") })
stats.GET("/charts/aircraft/day", func(c *gin.Context) { s.getChartAircraftOverTime(c, "day") })
+ // Depracacted
+ stats.GET("/seen/flights", s.getFlightsSeenMetrics)
+ stats.GET("/seen/aircraft", s.getAircraftSeenMetrics)
}
settings := api.Group("/settings")
@@ -152,7 +157,7 @@ func (s *APIServer) getFlightsSeenMetrics(c *gin.Context) {
// Today's flights count
var todayFlights int
err = s.pg.db.QueryRow(context.Background(),
- "SELECT COUNT(*) FROM aircraft_data WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayFlights)
+ "SELECT COUNT(*) FROM aircraft_data WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayFlights)
if err == nil {
stats["today_flights"] = todayFlights
}
@@ -184,7 +189,7 @@ func (s *APIServer) getAircraftSeenMetrics(c *gin.Context) {
// Today's aircraft count
var todayAircraft int
err = s.pg.db.QueryRow(context.Background(),
- "SELECT COUNT(DISTINCT hex) FROM aircraft_data WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayAircraft)
+ "SELECT COUNT(DISTINCT hex) FROM aircraft_data WHERE first_seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayAircraft)
if err == nil {
stats["today_aircraft"] = todayAircraft
}
@@ -257,22 +262,28 @@ func (s *APIServer) getInterestingMetrics(c *gin.Context) {
err := s.pg.db.QueryRow(context.Background(), "SELECT COUNT(*) FROM interesting_aircraft_seen").Scan(&interestingCount)
if err == nil {
stats["total_interesting"] = interestingCount
+ } else {
+ log.Error().Err(err).Msg("Error getting totalInterestingCount")
}
// Today's interesting aircraft count
var todayInterestingCount int
err = s.pg.db.QueryRow(context.Background(),
- "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE", tz).Scan(&todayInterestingCount)
+ "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE seen >= DATE_TRUNC('day', NOW(), $1)", tz).Scan(&todayInterestingCount)
if err == nil {
stats["today_interesting"] = todayInterestingCount
+ } else {
+ log.Error().Err(err).Msg("Error getting todayInterestingCount")
}
// Past hour interesting aircraft count
var hourInterestingCount int
err = s.pg.db.QueryRow(context.Background(),
- "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE first_seen >= NOW() - INTERVAL '1 hour'").Scan(&hourInterestingCount)
+ "SELECT COUNT(*) FROM interesting_aircraft_seen WHERE seen >= NOW() - INTERVAL '1 hour'").Scan(&hourInterestingCount)
if err == nil {
stats["hour_interesting"] = hourInterestingCount
+ } else {
+ log.Error().Err(err).Msg("Error getting hourInterestingCount")
}
c.JSON(http.StatusOK, stats)
@@ -673,11 +684,11 @@ func (s *APIServer) getTopAircraftTypes(c *gin.Context, period string, flightora
switch period {
case "year":
- timeFilter = `age(now(), first_seen) <= INTERVAL '1 year' AND`
+ timeFilter = `first_seen >= now() - INTERVAL '1 year' AND`
case "month":
- timeFilter = `age(now(), first_seen) <= INTERVAL '1 month' AND`
+ timeFilter = `first_seen >= now() - INTERVAL '1 month' AND`
case "day":
- timeFilter = `age(now(), first_seen) <= INTERVAL '1 day' AND`
+ timeFilter = `first_seen >= now() - INTERVAL '1 day' AND`
default:
timeFilter = ""
}
@@ -701,7 +712,7 @@ func (s *APIServer) getTopAircraftTypes(c *gin.Context, period string, flightora
SELECT t, Count(t) as count
FROM ` + innerQuery + `
GROUP BY t ORDER BY count DESC
- ) top_15
+ ) AS top_15
ORDER BY count DESC LIMIT 15`
rows, err := s.pg.db.Query(context.Background(), query)
@@ -1461,3 +1472,45 @@ func (s *APIServer) updateSettings(c *gin.Context) {
}
c.JSON(http.StatusOK, settings)
}
+
+func (s *APIServer) getRecentSeenMetrics(c *gin.Context) {
+ stats := gin.H{}
+ tz := s.getTimezone(c)
+
+ query := `
+ SELECT
+ COUNT(*) FILTER (WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE) as today_flights,
+ COUNT(*) FILTER (WHERE first_seen >= NOW() - INTERVAL '1 hour') as hour_flights,
+ COUNT(DISTINCT CASE WHEN DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE THEN hex END) as today_aircraft,
+ COUNT(DISTINCT CASE WHEN first_seen >= NOW() - INTERVAL '1 hour' THEN hex END) as hour_aircraft
+ FROM aircraft_data
+ WHERE first_seen >= CURRENT_DATE AT TIME ZONE $1 - INTERVAL '1 day'
+ `
+
+ var todayFlights, hourFlights, todayAircraft, hourAircraft int
+ err := s.pg.db.QueryRow(context.Background(), query, tz).Scan(&todayFlights, &hourFlights, &todayAircraft, &hourAircraft)
+ if err == nil {
+ stats["today_flights"] = todayFlights
+ stats["hour_flights"] = hourFlights
+ stats["today_aircraft"] = todayAircraft
+ stats["hour_aircraft"] = hourAircraft
+ }
+
+ c.JSON(http.StatusOK, stats)
+}
+
+func (s *APIServer) getTotalSeenMetrics(c *gin.Context) {
+ stats := gin.H{}
+
+ totalFlights, err := s.cachedStats.GetCachedStat("total_flights")
+ if err == nil {
+ stats["total_flights"] = totalFlights
+ }
+
+ totalAircraft, err := s.cachedStats.GetCachedStat("total_aircraft")
+ if err == nil {
+ stats["total_aircraft"] = totalAircraft
+ }
+
+ c.JSON(http.StatusOK, stats)
+}
diff --git a/core/cached-stats.go b/core/cached-stats.go
new file mode 100644
index 0000000..8731849
--- /dev/null
+++ b/core/cached-stats.go
@@ -0,0 +1,90 @@
+package main
+
+import (
+ "context"
+ "time"
+
+ "github.com/rs/zerolog/log"
+)
+
+type CachedStatsService struct {
+ pg *postgres
+}
+
+func NewCachedStatsService(pg *postgres) *CachedStatsService {
+ return &CachedStatsService{pg: pg}
+}
+
+func (s *CachedStatsService) GetCachedStat(key string) (int, error) {
+ var value int
+ err := s.pg.db.QueryRow(context.Background(),
+ "SELECT stat_value FROM cached_stats WHERE stat_key = $1", key).Scan(&value)
+ return value, err
+}
+
+func (s *CachedStatsService) UpdateCachedStat(key string, value int) error {
+ _, err := s.pg.db.Exec(context.Background(),
+ `INSERT INTO cached_stats (stat_key, stat_value, last_updated)
+ VALUES ($1, $2, NOW())
+ ON CONFLICT (stat_key)
+ DO UPDATE SET stat_value = $2, last_updated = NOW()`,
+ key, value)
+ return err
+}
+
+func (s *CachedStatsService) RefreshAllTotals() error {
+ log.Info().Msg("Refreshing cached all-time statistics...")
+
+ // Update total flights
+ var totalFlights int
+ err := s.pg.db.QueryRow(context.Background(),
+ "SELECT COUNT(*) FROM aircraft_data").Scan(&totalFlights)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to count total flights")
+ return err
+ }
+ err = s.UpdateCachedStat("total_flights", totalFlights)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to update cached total_flights")
+ return err
+ }
+
+ // Update total aircraft
+ var totalAircraft int
+ err = s.pg.db.QueryRow(context.Background(),
+ "SELECT COUNT(DISTINCT hex) FROM aircraft_data").Scan(&totalAircraft)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to count total aircraft")
+ return err
+ }
+ err = s.UpdateCachedStat("total_aircraft", totalAircraft)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to update cached total_aircraft")
+ return err
+ }
+
+ log.Info().
+ Int("total_flights", totalFlights).
+ Int("total_aircraft", totalAircraft).
+ Msg("Successfully refreshed cached statistics")
+
+ return nil
+}
+
+func (s *CachedStatsService) StartPeriodicRefresh(interval time.Duration) {
+ go func() {
+ // Initial refresh on startup
+ if err := s.RefreshAllTotals(); err != nil {
+ log.Error().Err(err).Msg("Initial cache refresh failed")
+ }
+
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ for range ticker.C {
+ if err := s.RefreshAllTotals(); err != nil {
+ log.Error().Err(err).Msg("Periodic cache refresh failed")
+ }
+ }
+ }()
+}
diff --git a/core/core.go b/core/core.go
index 9f44eaa..9434e3d 100644
--- a/core/core.go
+++ b/core/core.go
@@ -93,10 +93,14 @@ func main() {
os.Exit(1)
}
+ apiServer := NewAPIServer(pg)
+
+ log.Info().Msg("Starting cached statistics refresh service")
+ apiServer.cachedStats.StartPeriodicRefresh(24 * time.Hour)
+
// Start API server in a separate goroutine
log.Info().Msg("Starting API server")
go func() {
- apiServer := NewAPIServer(pg)
apiServer.Start()
}()
@@ -112,7 +116,11 @@ func main() {
if banner, err := os.ReadFile("../docs/logo/skystats_ascii.txt"); err == nil {
log.Info().Msg("\n" + string(banner))
}
- log.Info().Msg("Welcome to Skystats!")
+ if version == "dev" {
+ log.Info().Msgf("Welcome to Skystats! (build: %s • %s)", version, commit)
+ } else {
+ log.Info().Msgf("Welcome to Skystats %s!", version)
+ }
defer func() {
log.Info().Msg("Closing database connection")
diff --git a/migrations/000007_add_first_seen_index.down.sql b/migrations/000007_add_first_seen_index.down.sql
new file mode 100644
index 0000000..0381384
--- /dev/null
+++ b/migrations/000007_add_first_seen_index.down.sql
@@ -0,0 +1,2 @@
+-- remove first_seen index
+DROP INDEX IF EXISTS idx_aircraft_data_first_seen;
diff --git a/migrations/000007_add_first_seen_index.up.sql b/migrations/000007_add_first_seen_index.up.sql
new file mode 100644
index 0000000..0324cd3
--- /dev/null
+++ b/migrations/000007_add_first_seen_index.up.sql
@@ -0,0 +1,2 @@
+-- add first_seen index to speed up various queries
+CREATE INDEX IF NOT EXISTS idx_aircraft_data_first_seen ON aircraft_data (first_seen DESC);
diff --git a/migrations/000008_add_cached_stats.down.sql b/migrations/000008_add_cached_stats.down.sql
new file mode 100644
index 0000000..bb47cd2
--- /dev/null
+++ b/migrations/000008_add_cached_stats.down.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS cached_stats;
diff --git a/migrations/000008_add_cached_stats.up.sql b/migrations/000008_add_cached_stats.up.sql
new file mode 100644
index 0000000..b346ecc
--- /dev/null
+++ b/migrations/000008_add_cached_stats.up.sql
@@ -0,0 +1,12 @@
+-- Create table for cached statistics
+CREATE TABLE cached_stats (
+ stat_key VARCHAR(50) PRIMARY KEY,
+ stat_value BIGINT NOT NULL,
+ last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Insert initial values for all-time totals
+INSERT INTO cached_stats (stat_key, stat_value, last_updated)
+VALUES
+ ('total_flights', (SELECT COUNT(*) FROM aircraft_data), NOW()),
+ ('total_aircraft', (SELECT COUNT(DISTINCT hex) FROM aircraft_data), NOW());
diff --git a/migrations/000009_add_above_index.down.sql b/migrations/000009_add_above_index.down.sql
new file mode 100644
index 0000000..c1b8bb7
--- /dev/null
+++ b/migrations/000009_add_above_index.down.sql
@@ -0,0 +1 @@
+DROP INDEX IF EXISTS idx_aircraft_data_last_seen_distance;
diff --git a/migrations/000009_add_above_index.up.sql b/migrations/000009_add_above_index.up.sql
new file mode 100644
index 0000000..e4e4acb
--- /dev/null
+++ b/migrations/000009_add_above_index.up.sql
@@ -0,0 +1,2 @@
+CREATE INDEX idx_aircraft_data_last_seen_distance
+ON aircraft_data (last_seen DESC, last_seen_distance);
diff --git a/web/src/components/InterestingAircraft.svelte b/web/src/components/InterestingAircraft.svelte
index edffa44..039eb14 100644
--- a/web/src/components/InterestingAircraft.svelte
+++ b/web/src/components/InterestingAircraft.svelte
@@ -1,5 +1,5 @@
diff --git a/web/src/components/MetricFlightsSeen.svelte b/web/src/components/MetricFlightsSeen.svelte
index a116dbf..e8d0ebe 100644
--- a/web/src/components/MetricFlightsSeen.svelte
+++ b/web/src/components/MetricFlightsSeen.svelte
@@ -5,13 +5,12 @@
import SkeletonMetrics from './SkeletonMetrics.svelte';
let data = {};
- let endpoint = 'api/stats/seen/flights'
+ let endpoint = 'api/stats/seen/recent'
let loading = true;
let error = null;
- let interval = null;
-
- async function fetchData() {
+ let recentInterval = null;
+ async function fetchRecentData() {
try {
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone;
const response = await fetch(`${endpoint}?tz=${encodeURIComponent(tz)}`);
@@ -19,7 +18,7 @@
throw new Error(`${response.status}`);
}
const result = await response.json();
- data = result;
+ data = { ...data, ...result };
error = null;
} catch (err) {
error = err.message;
@@ -28,14 +27,29 @@
}
}
+ async function fetchTotalData() {
+ try {
+ const response = await fetch(`api/stats/seen/totals`);
+ if (!response.ok) {
+ throw new Error(`${response.status}`);
+ }
+ const result = await response.json();
+ data = { ...data, ...result };
+ error = null;
+ } catch (err) {
+ error = err.message;
+ }
+ }
+
onMount(() => {
- fetchData();
- interval = setInterval(fetchData, 2000);
+ fetchRecentData();
+ fetchTotalData();
+ recentInterval = setInterval(fetchRecentData, 2000);
})
onDestroy(() => {
- if (interval) {
- clearInterval(interval);
+ if (recentInterval) {
+ clearInterval(recentInterval);
}
});
diff --git a/web/src/components/MotionStats.svelte b/web/src/components/MotionStats.svelte
index d081c3d..d0dbbd6 100644
--- a/web/src/components/MotionStats.svelte
+++ b/web/src/components/MotionStats.svelte
@@ -1,6 +1,6 @@