Enable long-term storage via Apache Arrow + R2

README.md

@@ -18,7 +18,7 @@ Counterscale is free, open source software made available under the MIT license.
 
 ## Limitations
 
-Counterscale is powered primarily by Cloudflare Workers and [Workers Analytics Engine](https://developers.cloudflare.com/analytics/analytics-engine/). As of February 2025, Workers Analytics Engine has _maximum 90 days retention_, which means Counterscale can only show the last 90 days of recorded data.
+Counterscale is powered primarily by Cloudflare Workers and [Workers Analytics Engine](https://developers.cloudflare.com/analytics/analytics-engine/). As of February 2025, Workers Analytics Engine has _maximum 90 days retention_, which means Counterscale can only show the last 90 days of recorded data. We do, however, provide long term storage of your data in an R2 bucket using Apache Arrow files. This long term storage is enabled by default and can be disabled using the CLI.
 
 ## Installation
 
@@ -254,6 +254,33 @@ Update/roll the password:
 npx @counterscale/cli@latest auth roll
 ```
 
+#### `storage`
+
+Manage long term storage settings for your Counterscale deployment.
+
+```bash
+npx @counterscale/cli@latest storage [subcommand]
+```
+
+Available subcommands:
+
+- `enable` - Enable storage for your Counterscale deployment
+- `disable` - Disable storage for your Counterscale deployment
+
+##### Examples:
+
+Enable storage:
+
+```bash
+npx @counterscale/cli@latest storage enable
+```
+
+Disable storage:
+
+```bash
+npx @counterscale/cli@latest storage disable
+```
+
 ## Development
 
 See [Contributing](CONTRIBUTING.md) for information on how to get started.

package.json

@@ -23,5 +23,10 @@
     "engines": {
         "node": ">=20",
         "pnpm": ">=9"
+    },
+    "pnpm": {
+        "patchedDependencies": {
+            "apache-arrow": "patches/apache-arrow.patch"
+        }
     }
 }

packages/cli/src/commands/storage.ts

@@ -0,0 +1,51 @@
+import { CloudflareClient } from "../lib/cloudflare.js";
+import { getServerPkgDir } from "../lib/config.js";
+import path from "path";
+
+export async function enableStorage() {
+    const serverPkgDir = getServerPkgDir();
+    const configPath = path.join(serverPkgDir, "wrangler.json");
+    const cloudflare = new CloudflareClient(configPath);
+
+    try {
+        console.log("Enabling storage...");
+
+        const success = await cloudflare.setCloudflareSecrets({
+            CF_STORAGE_ENABLED: "true",
+        });
+
+        if (success) {
+            console.log("✅ Storage enabled successfully!");
+        } else {
+            console.error("❌ Failed to enable storage");
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error("❌ Error enabling storage:", error);
+        process.exit(1);
+    }
+}
+
+export async function disableStorage() {
+    const serverPkgDir = getServerPkgDir();
+    const configPath = path.join(serverPkgDir, "wrangler.json");
+    const cloudflare = new CloudflareClient(configPath);
+
+    try {
+        console.log("Disabling storage...");
+
+        const success = await cloudflare.setCloudflareSecrets({
+            CF_STORAGE_ENABLED: "false",
+        });
+
+        if (success) {
+            console.log("✅ Storage disabled successfully!");
+        } else {
+            console.error("❌ Failed to disable storage");
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error("❌ Error disabling storage:", error);
+        process.exit(1);
+    }
+}

packages/cli/src/index.ts

@@ -6,6 +6,7 @@ import { getTitle } from "./lib/ui.js";
 import { getServerPkgDir } from "./lib/config.js";
 import { install } from "./commands/install.js";
 import { enableAuth, disableAuth, updatePassword } from "./commands/auth.js";
+import { enableStorage, disableStorage } from "./commands/storage.js";
 import { envCommand, SECRETS_BY_ALIAS } from "./commands/env.js";
 
 import { hideBin } from "yargs/helpers";
@@ -93,6 +94,47 @@ const parser = yargs(hideBin(process.argv))
             }
         },
     )
+    .command(
+        "storage",
+        "Manage long term storage settings",
+        (yargs) => {
+            const storageYargs = yargs
+                .command(
+                    "enable",
+                    "Enable long term data storage for your Counterscale data",
+                    {},
+                    enableStorage,
+                )
+                .command(
+                    "disable",
+                    "Disable long term data storage for your Counterscale data",
+                    {},
+                    disableStorage,
+                )
+                .help();
+            return storageYargs;
+        },
+        async (argv) => {
+            // Show help if no subcommand was provided
+            if (argv._.length === 1) {
+                const storageYargs = yargs(hideBin(process.argv))
+                    .command(
+                        "enable",
+                        "Enable storage for your Counterscale deployment",
+                        {},
+                        enableStorage,
+                    )
+                    .command(
+                        "disable",
+                        "Disable storage for your Counterscale deployment",
+                        {},
+                        disableStorage,
+                    )
+                    .help();
+                storageYargs.showHelp();
+            }
+        },
+    )
     .command(
         "env [secret]",
         "Update environment secrets",

packages/server/.dev.vars.example

@@ -5,3 +5,4 @@ CF_ACCOUNT_ID=''
 CF_PASSWORD_HASH=''
 CF_JWT_SECRET=''
 CF_AUTH_ENABLED=''
+CF_STORAGE_ENABLED=''

packages/server/app/analytics/query.ts

@@ -464,6 +464,84 @@ export class AnalyticsEngineAPI {
         return returnPromise;
     }
 
+    async getAllCountsByAllColumnsForAllSites(
+        columns: (keyof typeof ColumnMappings)[],
+        startDateTime: Date,
+        endDateTime: Date,
+        tz?: string,
+    ): Promise<Map<string[], AnalyticsCountResult>> {
+        const columnsStr = columns.map((c) => ColumnMappings[c]).join(", ");
+        const columnsStrWithAliases = columns
+            .map((c) => ColumnMappings[c] + " as " + c)
+            .join(", ");
+
+        const startDateTimeSql = dayjs(startDateTime)
+            .tz(tz)
+            .utc()
+            .format("YYYY-MM-DD HH:mm:ss");
+        const endDateTimeSql = dayjs(endDateTime)
+            .tz(tz)
+            .utc()
+            .format("YYYY-MM-DD HH:mm:ss");
+
+        const query = `
+            SELECT 
+                timestamp,
+                SUM(_sample_interval) as count,
+                ${ColumnMappings.siteId} as siteId, 
+                ${ColumnMappings.newVisitor} as isVisitor, 
+                ${ColumnMappings.bounce} as isBounce,
+                ${columnsStrWithAliases}
+            FROM metricsDataset
+            WHERE timestamp >= toDateTime('${startDateTimeSql}') AND timestamp < toDateTime('${endDateTimeSql}')
+            GROUP BY timestamp,
+                ${ColumnMappings.siteId}, 
+                ${ColumnMappings.newVisitor}, 
+                ${ColumnMappings.bounce}, 
+                ${columnsStr}
+            ORDER BY count DESC
+        `;
+
+        type SelectionSet = {
+            date: string;
+            count: number;
+            isVisitor: number;
+            isBounce: number;
+        } & {
+            [K in keyof typeof ColumnMappings]: string;
+        };
+
+        return this.query(query).then(async (response) => {
+            if (!response.ok) {
+                throw new Error(response.status + response.statusText);
+            }
+
+            const responseData =
+                (await response.json()) as AnalyticsQueryResult<SelectionSet>;
+
+
+            return responseData.data.reduce((acc, row) => {
+                // key is the comma joined string of siteId + all columns
+                const key = [
+                    row.date,
+                    row.siteId,
+                    ...columns.map((c) => String(row[c]).trim()),
+                ];
+
+                if (!acc.has(key)) {
+                    acc.set(key, {
+                        views: 0,
+                        visitors: 0,
+                        bounces: 0,
+                    } as AnalyticsCountResult);
+                }
+
+                accumulateCountsFromRowResult(acc.get(key)!, row);
+                return acc;
+            }, new Map<string[], AnalyticsCountResult>());
+        });
+    }
+
     async getAllCountsByColumn<T extends keyof typeof ColumnMappings>(
         siteId: string,
         column: T,

packages/server/app/routes/__tests__/cache.test.tsx

@@ -2,7 +2,7 @@
 import { vi, test, describe, beforeEach, afterEach, expect } from "vitest";
 import "vitest-dom/extend-expect";
 
-import { loader } from "../cache";
+import { CacheLoaderBody, loader } from "../cache";
 
 describe("Cache route", () => {
     beforeEach(() => {
@@ -54,7 +54,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should return a hit count > 0 for returning visitors
             expect(data.ht).toBeGreaterThan(0);
@@ -81,7 +81,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should be the first hit (new visitor) because a new day began
             expect(data.ht).toBe(1);
@@ -104,7 +104,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should be the first hit (new visitor) because > 30 days passed
             expect(data.ht).toBe(1);
@@ -127,7 +127,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should be the first hit (new visitor) because > 24 hours passed
             expect(data.ht).toBe(1);
@@ -156,7 +156,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should return a hit count > 0 for returning visitors
             expect(data.ht).toBe(2);
@@ -189,7 +189,7 @@ describe("Cache route", () => {
             const response = await loader({ request } as any);
 
             // Verify the content of the response
-            const data = await response.json();
+            const data = await response.json() as CacheLoaderBody;
 
             // Should be the third hit (returning visitor)
             expect(data.ht).toBe(3);

packages/server/app/routes/cache.tsx

@@ -1,6 +1,10 @@
 import type { LoaderFunctionArgs } from "react-router";
 import { handleCacheHeaders } from "~/analytics/collect";
 
+export type CacheLoaderBody = {
+  ht: number;
+}
+
 /**
  * Loader function for the /cache route.
  *
@@ -17,7 +21,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
     // Return the hit count to the client
     const payload = {
         ht: hits, // Number of hits in the current session (hit type)
-    };
+    } satisfies CacheLoaderBody;
 
     // Return the JSON payload with the appropriate Last-Modified header
     return new Response(JSON.stringify(payload), {

packages/server/package.json

@@ -38,6 +38,7 @@
         "@radix-ui/react-slot": "^1.1.0",
         "@react-router/cloudflare": "7.1.1",
         "@types/jsonwebtoken": "^9.0.10",
+        "apache-arrow": "^21.1.0",
         "bcryptjs": "^3.0.2",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.1",
@@ -59,6 +60,7 @@
         "@react-router/dev": "7.7.1",
         "@react-router/fs-routes": "7.7.1",
         "@testing-library/react": "^16.2.0",
+        "@types/node": "^22.10.2",
         "@types/react": "^18.3.3",
         "@types/react-dom": "^18.3.0",
         "@types/recharts": "^1.8.29",