fix: api performance improvements

core/api.go

@@ -14,9 +14,10 @@ import (
 )
 
 type APIServer struct {
-	pg       *postgres
-	port     string
-	settings *SettingsService
+	pg          *postgres
+	port        string
+	settings    *SettingsService
+	cachedStats *CachedStatsService
 }
 
 func NewAPIServer(pg *postgres) *APIServer {
@@ -25,9 +26,10 @@ func NewAPIServer(pg *postgres) *APIServer {
 		port = "8080"
 	}
 	return &APIServer{
-		pg:       pg,
-		port:     port,
-		settings: NewSettingsService(pg),
+		pg:          pg,
+		port:        port,
+		settings:    NewSettingsService(pg),
+		cachedStats: NewCachedStatsService(pg),
 	}
 }
 
@@ -79,8 +81,8 @@ func (s *APIServer) Start() {
 		{
 			stats.GET("/above", s.getAboveStats)
 
-			stats.GET("/seen/flights", s.getFlightsSeenMetrics)
-			stats.GET("/seen/aircraft", s.getAircraftSeenMetrics)
+			stats.GET("/seen/recent", s.getRecentSeenMetrics)
+			stats.GET("/seen/totals", s.getTotalSeenMetrics)
 
 			stats.GET("/routes/metrics", s.getRouteMetrics)
 			stats.GET("/routes/airlines", s.getTopAirlines)
@@ -119,6 +121,9 @@ func (s *APIServer) Start() {
 			stats.GET("/charts/aircraft/month", func(c *gin.Context) { s.getChartAircraftOverTime(c, "month") })
 			stats.GET("/charts/aircraft/day", func(c *gin.Context) { s.getChartAircraftOverTime(c, "day") })
 
+			// Depracacted
+			stats.GET("/seen/flights", s.getFlightsSeenMetrics)
+			stats.GET("/seen/aircraft", s.getAircraftSeenMetrics)
 		}
 
 		settings := api.Group("/settings")
@@ -1461,3 +1466,45 @@ func (s *APIServer) updateSettings(c *gin.Context) {
 	}
 	c.JSON(http.StatusOK, settings)
 }
+
+func (s *APIServer) getRecentSeenMetrics(c *gin.Context) {
+	stats := gin.H{}
+	tz := s.getTimezone(c)
+
+	query := `
+		SELECT
+			COUNT(*) FILTER (WHERE DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE) as today_flights,
+			COUNT(*) FILTER (WHERE first_seen >= NOW() - INTERVAL '1 hour') as hour_flights,
+			COUNT(DISTINCT CASE WHEN DATE(first_seen AT TIME ZONE $1) = CURRENT_DATE THEN hex END) as today_aircraft,
+			COUNT(DISTINCT CASE WHEN first_seen >= NOW() - INTERVAL '1 hour' THEN hex END) as hour_aircraft
+		FROM aircraft_data
+		WHERE first_seen >= CURRENT_DATE AT TIME ZONE $1 - INTERVAL '1 day'
+	`
+
+	var todayFlights, hourFlights, todayAircraft, hourAircraft int
+	err := s.pg.db.QueryRow(context.Background(), query, tz).Scan(&todayFlights, &hourFlights, &todayAircraft, &hourAircraft)
+	if err == nil {
+		stats["today_flights"] = todayFlights
+		stats["hour_flights"] = hourFlights
+		stats["today_aircraft"] = todayAircraft
+		stats["hour_aircraft"] = hourAircraft
+	}
+
+	c.JSON(http.StatusOK, stats)
+}
+
+func (s *APIServer) getTotalSeenMetrics(c *gin.Context) {
+	stats := gin.H{}
+
+	totalFlights, err := s.cachedStats.GetCachedStat("total_flights")
+	if err == nil {
+		stats["total_flights"] = totalFlights
+	}
+
+	totalAircraft, err := s.cachedStats.GetCachedStat("total_aircraft")
+	if err == nil {
+		stats["total_aircraft"] = totalAircraft
+	}
+
+	c.JSON(http.StatusOK, stats)
+}

core/cached-stats.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+type CachedStatsService struct {
+	pg *postgres
+}
+
+func NewCachedStatsService(pg *postgres) *CachedStatsService {
+	return &CachedStatsService{pg: pg}
+}
+
+func (s *CachedStatsService) GetCachedStat(key string) (int, error) {
+	var value int
+	err := s.pg.db.QueryRow(context.Background(),
+		"SELECT stat_value FROM cached_stats WHERE stat_key = $1", key).Scan(&value)
+	return value, err
+}
+
+func (s *CachedStatsService) UpdateCachedStat(key string, value int) error {
+	_, err := s.pg.db.Exec(context.Background(),
+		`INSERT INTO cached_stats (stat_key, stat_value, last_updated)
+		VALUES ($1, $2, NOW())
+		ON CONFLICT (stat_key)
+		DO UPDATE SET stat_value = $2, last_updated = NOW()`,
+		key, value)
+	return err
+}
+
+func (s *CachedStatsService) RefreshAllTotals() error {
+	log.Info().Msg("Refreshing cached all-time statistics...")
+
+	// Update total flights
+	var totalFlights int
+	err := s.pg.db.QueryRow(context.Background(),
+		"SELECT COUNT(*) FROM aircraft_data").Scan(&totalFlights)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to count total flights")
+		return err
+	}
+	err = s.UpdateCachedStat("total_flights", totalFlights)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to update cached total_flights")
+		return err
+	}
+
+	// Update total aircraft
+	var totalAircraft int
+	err = s.pg.db.QueryRow(context.Background(),
+		"SELECT COUNT(DISTINCT hex) FROM aircraft_data").Scan(&totalAircraft)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to count total aircraft")
+		return err
+	}
+	err = s.UpdateCachedStat("total_aircraft", totalAircraft)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to update cached total_aircraft")
+		return err
+	}
+
+	log.Info().
+		Int("total_flights", totalFlights).
+		Int("total_aircraft", totalAircraft).
+		Msg("Successfully refreshed cached statistics")
+
+	return nil
+}
+
+func (s *CachedStatsService) StartPeriodicRefresh(interval time.Duration) {
+	go func() {
+		// Initial refresh on startup
+		if err := s.RefreshAllTotals(); err != nil {
+			log.Error().Err(err).Msg("Initial cache refresh failed")
+		}
+
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+
+		for range ticker.C {
+			if err := s.RefreshAllTotals(); err != nil {
+				log.Error().Err(err).Msg("Periodic cache refresh failed")
+			}
+		}
+	}()
+}

core/core.go

@@ -93,10 +93,14 @@ func main() {
 		os.Exit(1)
 	}
 
+	apiServer := NewAPIServer(pg)
+
+	log.Info().Msg("Starting cached statistics refresh service")
+	apiServer.cachedStats.StartPeriodicRefresh(24 * time.Hour)
+
 	// Start API server in a separate goroutine
 	log.Info().Msg("Starting API server")
 	go func() {
-		apiServer := NewAPIServer(pg)
 		apiServer.Start()
 	}()
 

migrations/000007_add_cached_stats.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS cached_stats;

migrations/000007_add_cached_stats.up.sql

@@ -0,0 +1,12 @@
+-- Create table for cached statistics
+CREATE TABLE cached_stats (
+    stat_key VARCHAR(50) PRIMARY KEY,
+    stat_value BIGINT NOT NULL,
+    last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Insert initial values for all-time totals
+INSERT INTO cached_stats (stat_key, stat_value, last_updated)
+VALUES
+    ('total_flights', (SELECT COUNT(*) FROM aircraft_data), NOW()),
+    ('total_aircraft', (SELECT COUNT(DISTINCT hex) FROM aircraft_data), NOW());

migrations/000008_add_above_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS idx_aircraft_data_last_seen_distance;

migrations/000008_add_above_index.up.sql

@@ -0,0 +1,2 @@
+CREATE INDEX idx_aircraft_data_last_seen_distance
+ON aircraft_data (last_seen DESC, last_seen_distance);

web/src/components/InterestingAircraft.svelte

@@ -1,17 +1,15 @@
 <script>
-    import { onMount, onDestroy } from 'svelte'
+    import { onMount } from 'svelte'
     import { settings, refreshInterestingData } from '../stores/settings';
 
     export let endpoint;
     export let title;
     export let icon;
     export let aircraftType;
 
-    let refreshRate = 10000
     let data = [];
     let loading = true;
     let error = null;
-    let interval = null;
     let selectedAircraft = null;
     let imageLoadingStates = {
         image1: true,
@@ -53,15 +51,8 @@
 
     onMount(() => {
         fetchData();
-        interval = setInterval(fetchData, refreshRate)
     })
 
-    onDestroy(() => {
-        if (interval) {
-            clearInterval(interval)
-        }
-    });
-
     // Refresh when settings change
     $: if ($refreshInterestingData) {
         fetchData();

web/src/components/MetricAircraftSeen.svelte

@@ -4,23 +4,21 @@
     import NumberFlow from '@number-flow/svelte'
     import SkeletonMetrics from './SkeletonMetrics.svelte';
 
-
     let data = {};
-    let endpoint = 'api/stats/seen/aircraft'
+    let endpoint = 'api/stats/seen/recent'
     let loading = true;
     let error = null;
-    let interval = null;
-
-    async function fetchData() {
+    let recentInterval = null;
 
+    async function fetchRecentData() {
         try {
             const tz = Intl.DateTimeFormat().resolvedOptions().timeZone;
             const response = await fetch(`${endpoint}?tz=${encodeURIComponent(tz)}`);
             if (!response.ok) {
                 throw new Error(`${response.status}`);
             }
             const result = await response.json();
-            data = result;
+            data = { ...data, ...result };
             error = null;
         } catch (err) {
             error = err.message;
@@ -29,14 +27,29 @@
         }
     }
 
+    async function fetchTotalData() {
+        try {
+            const response = await fetch(`api/stats/seen/totals`);
+            if (!response.ok) {
+                throw new Error(`${response.status}`);
+            }
+            const result = await response.json();
+            data = { ...data, ...result };
+            error = null;
+        } catch (err) {
+            error = err.message;
+        }
+    }
+
     onMount(() => {
-        fetchData();
-        interval = setInterval(fetchData, 2000);
+        fetchRecentData();
+        fetchTotalData();
+        recentInterval = setInterval(fetchRecentData, 2000);
     })
 
     onDestroy(() => {
-        if (interval) {
-            clearInterval(interval);
+        if (recentInterval) {
+            clearInterval(recentInterval);
         }
     });
 </script>

web/src/components/MetricFlightsSeen.svelte

@@ -5,21 +5,20 @@
     import SkeletonMetrics from './SkeletonMetrics.svelte';
 
     let data = {};
-    let endpoint = 'api/stats/seen/flights'
+    let endpoint = 'api/stats/seen/recent'
     let loading = true;
     let error = null;
-    let interval = null;
-
-    async function fetchData() {
+    let recentInterval = null;
 
+    async function fetchRecentData() {
         try {
             const tz = Intl.DateTimeFormat().resolvedOptions().timeZone;
             const response = await fetch(`${endpoint}?tz=${encodeURIComponent(tz)}`);
             if (!response.ok) {
                 throw new Error(`${response.status}`);
             }
             const result = await response.json();
-            data = result;
+            data = { ...data, ...result };
             error = null;
         } catch (err) {
             error = err.message;
@@ -28,14 +27,29 @@
         }
     }
 
+    async function fetchTotalData() {
+        try {
+            const response = await fetch(`api/stats/seen/totals`);
+            if (!response.ok) {
+                throw new Error(`${response.status}`);
+            }
+            const result = await response.json();
+            data = { ...data, ...result };
+            error = null;
+        } catch (err) {
+            error = err.message;
+        }
+    }
+
     onMount(() => {
-        fetchData();
-        interval = setInterval(fetchData, 2000);
+        fetchRecentData();
+        fetchTotalData();
+        recentInterval = setInterval(fetchRecentData, 2000);
     })
 
     onDestroy(() => {
-        if (interval) {
-            clearInterval(interval);
+        if (recentInterval) {
+            clearInterval(recentInterval);
         }
     });
 </script>