diff --git a/.dev.vars.example b/.dev.vars.example index db78cfa..6ee5d84 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -3,4 +3,5 @@ MATOMO_SITE_ID=1 MATOMO_TIMEOUT_MS=5000 LOG_LEVEL=debug USER_AGENT_ALLOWLIST_REGEX=(?:ChatGPT-User|MistralAI-User|Gemini-Deep-Research|Claude-User|Perplexity-User|Google-NotebookLM|Devin) +URL_EXCLUDE_REGEX=^[^?]+\.(?:css|js|mjs|map|json|xml|webmanifest|manifest|png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|woff2?|ttf|otf|eot|rss|atom|wasm|txt)(?:\?|$) DOCUMENT_REGEX=^[^?]+\.(?:pdf|docx?|xlsx?|pptx?|csv|json|txt|xml|epub|mobi|azw3|mp3|mp4|mpe?g|webm|mov|avi|ogg|wav|flac|zip|gz|gzip|tgz|tar|bz2|tbz|7z|rar|dmg|exe|msi|apk|jar|md5|sig)(?:\?|$) diff --git a/README.md b/README.md index 5b69968..708bf7e 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,32 @@ Cloudflare Worker (TypeScript, Node 24 tooling) that sits inline on your zone, p - `MATOMO_URL` (required): Base Matomo URL, e.g. `https://analytics.example.com`. - `MATOMO_SITE_ID` (required): Matomo site ID (integer). - `MATOMO_TIMEOUT_MS` (optional, default `5000`): HTTP timeout in ms for Matomo calls. -- `DOCUMENT_REGEX` (optional): Case-insensitive regex to detect downloads; matching URLs add `download=` to Matomo payloads. Defaults to common document/media/archive extensions. +- `DOCUMENT_REGEX` (optional): Case-insensitive regex to detect downloads; matching URLs add `download=` to Matomo payloads. This regex runs against the full URL (`protocol://host/path?query`) and defaults to a modern/common set of extensions: + - Documents: `.pdf`, `.doc`, `.docx`, `.xls`, `.xlsx`, `.ppt`, `.pptx` + - Data/text: `.csv`, `.json`, `.txt`, `.xml` + - Ebooks: `.epub`, `.mobi`, `.azw3` + - Media (audio/video): `.mp3`, `.mp4`, `.mpeg`, `.mpg`, `.webm`, `.mov`, `.avi`, `.ogg`, `.wav`, `.flac` + - Archives: `.zip`, `.gz`, `.gzip`, `.tgz`, `.tar`, `.bz2`, `.tbz`, `.7z`, `.rar` + - Installers/binaries: `.dmg`, `.exe`, `.msi`, `.apk`, `.jar` + - Hashes/signatures: `.md5`, `.sig` + + Example: `^[^?]+\\.(?:pdf|zip|docx?)(?:\\?|$)` + - `LOG_LEVEL` (optional, default `warn`): `silent|error|warn|info|debug`. - `USER_AGENT_ALLOWLIST_REGEX` (optional): Case-insensitive regex to permit user agents; non-matching entries are skipped. Defaults to an allowlist for `ChatGPT-User|MistralAI-User|Gemini-Deep-Research|Claude-User|Perplexity-User|Google-NotebookLM|Devin`. +- `URL_EXCLUDE_REGEX` (optional): Case-insensitive regex to skip tracking for matching URLs. This regex runs against the full URL (`protocol://host/path?query`) and defaults to excluding common static assets and non-page resources: + - Frontend assets: `.css`, `.js`, `.mjs` + - Source maps: `.map` + - Data/config: `.json`, `.xml`, `.webmanifest`, `.manifest` + - Feeds: `.rss`, `.atom` + - WebAssembly: `.wasm` + - Text: `.txt` + - Images: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.avif`, `.svg`, `.ico`, `.bmp`, `.tif`, `.tiff` + - Fonts: `.woff`, `.woff2`, `.ttf`, `.otf`, `.eot` + + Example: `^[^?]+\\.(?:css|js|png)(?:\\?|$)` + + Note: If a URL matches `URL_EXCLUDE_REGEX`, it is skipped even if it also matches `DOCUMENT_REGEX` (i.e. it will not be tracked as a download). Bind these as plain text environment variables in your Worker (e.g., Wrangler `vars`). @@ -27,7 +50,7 @@ Wrangler bundles the TypeScript entry for you; no manual build is required for ` - Install Wrangler (e.g., `npm install -g wrangler` or `npx wrangler --version` to use npx). - Copy `.dev.vars.example` to `.dev.vars` and set your local values (these are only for `wrangler dev --local`): - - `MATOMO_URL`, `MATOMO_SITE_ID`, `MATOMO_TIMEOUT_MS`, `LOG_LEVEL`, `USER_AGENT_ALLOWLIST_REGEX`, `DOCUMENT_REGEX` + - `MATOMO_URL`, `MATOMO_SITE_ID`, `MATOMO_TIMEOUT_MS`, `LOG_LEVEL`, `USER_AGENT_ALLOWLIST_REGEX`, `URL_EXCLUDE_REGEX`, `DOCUMENT_REGEX` - Start local dev (serves on http://localhost:8787 by default): ```sh @@ -78,7 +101,7 @@ The Worker simply calls `fetch(request)` to reach your origin and separately pos - Receives each incoming request, proxies to origin with `fetch`, and returns the origin response. - Measures server time (`pf_srv` in seconds), status, and response bytes from `Content-Length` when present. - Builds a Matomo payload with `idsite`, `rec:1`, `recMode:1`, `url`, `source:'Cloudflare'`, `cdt` (UTC `YYYY-MM-DD HH:mm:ss`), and `ua`. -- Detects downloads via `DOCUMENT_REGEX` and user-agent allowlist via `USER_AGENT_ALLOWLIST_REGEX`; disallowed UAs are skipped. +- Skips tracking when `URL_EXCLUDE_REGEX` matches; detects downloads via `DOCUMENT_REGEX`; disallowed UAs are skipped by `USER_AGENT_ALLOWLIST_REGEX`. - Sends a single Matomo hit asynchronously via `waitUntil` to `/matomo.php` (standard tracking API) with timeout. ## Logging diff --git a/src/config.ts b/src/config.ts index e9119d6..c0e3519 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,6 +25,8 @@ const defaultAllowlistPattern = `(?:${defaultUserAgentPatterns .join('|')})`; const defaultDocumentPattern = '^[^?]+\\.(?:pdf|docx?|xlsx?|pptx?|csv|json|txt|xml|epub|mobi|azw3|mp3|mp4|mpe?g|webm|mov|avi|ogg|wav|flac|zip|gz|gzip|tgz|tar|bz2|tbz|7z|rar|dmg|exe|msi|apk|jar|md5|sig)(?:\\?|$)'; +const defaultUrlExcludePattern = + '^[^?]+\\.(?:css|js|mjs|map|json|xml|webmanifest|manifest|png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|woff2?|ttf|otf|eot|rss|atom|wasm|txt)(?:\\?|$)'; export function getConfig( env: Partial & Record = process.env @@ -47,8 +49,10 @@ export function getConfig( const logLevel = (env.LOG_LEVEL || 'warn').toLowerCase() as LogLevel; const allowlistPattern = env.USER_AGENT_ALLOWLIST_REGEX || defaultAllowlistPattern; + const urlExcludePattern = env.URL_EXCLUDE_REGEX || defaultUrlExcludePattern; const documentPattern = env.DOCUMENT_REGEX || defaultDocumentPattern; let userAgentAllowlistRegex: RegExp | undefined; + let urlExcludeRegex: RegExp | undefined; let documentRegex: RegExp | undefined; try { userAgentAllowlistRegex = new RegExp(allowlistPattern, 'i'); @@ -56,6 +60,12 @@ export function getConfig( const message = err instanceof Error ? err.message : String(err); throw new Error(`Invalid USER_AGENT_ALLOWLIST_REGEX: ${message}`); } + try { + urlExcludeRegex = new RegExp(urlExcludePattern, 'i'); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Invalid URL_EXCLUDE_REGEX: ${message}`); + } try { documentRegex = new RegExp(documentPattern, 'i'); } catch (err) { @@ -69,6 +79,7 @@ export function getConfig( matomoTimeoutMs, logLevel, userAgentAllowlistRegex, + urlExcludeRegex, documentRegex }; } diff --git a/src/index.ts b/src/index.ts index ef5ae23..4f19d42 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ const trackRequest = async ( new Date(Date.now() - durationMs) ); if (!payload) { - log.debug('Tracking skipped (user agent not allowed)'); + log.debug('Tracking skipped'); return; } await sendMatomoHit( diff --git a/src/matomo.ts b/src/matomo.ts index a6b1294..2a75477 100644 --- a/src/matomo.ts +++ b/src/matomo.ts @@ -18,6 +18,9 @@ export function buildMatomoPayload( } const url = request.url; + if (config.urlExcludeRegex && config.urlExcludeRegex.test(url)) { + return null; + } const ua = request.headers.get('user-agent') || ''; if (!isUserAgentAllowed(ua, config.userAgentAllowlistRegex)) { return null; diff --git a/src/types.ts b/src/types.ts index 8ff80ac..61ea58e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,6 +13,7 @@ export interface Env { MATOMO_TIMEOUT_MS?: string; LOG_LEVEL?: LogLevel; USER_AGENT_ALLOWLIST_REGEX?: string; + URL_EXCLUDE_REGEX?: string; DOCUMENT_REGEX?: string; [key: string]: string | undefined; } @@ -23,6 +24,7 @@ export interface MatomoConfig { matomoTimeoutMs: number; logLevel: LogLevel; userAgentAllowlistRegex?: RegExp; + urlExcludeRegex?: RegExp; documentRegex?: RegExp; } diff --git a/tests/config.test.ts b/tests/config.test.ts index 9c20298..3977088 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -18,6 +18,9 @@ describe('getConfig', () => { expect(config.userAgentAllowlistRegex).toEqual( /(?:ChatGPT-User|MistralAI-User|Gemini-Deep-Research|Claude-User|Perplexity-User|Google-NotebookLM|Devin)/i ); + expect(config.urlExcludeRegex).toEqual( + /^[^?]+\.(?:css|js|mjs|map|json|xml|webmanifest|manifest|png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|woff2?|ttf|otf|eot|rss|atom|wasm|txt)(?:\?|$)/i + ); expect(config.documentRegex).toEqual( /^[^?]+\.(?:pdf|docx?|xlsx?|pptx?|csv|json|txt|xml|epub|mobi|azw3|mp3|mp4|mpe?g|webm|mov|avi|ogg|wav|flac|zip|gz|gzip|tgz|tar|bz2|tbz|7z|rar|dmg|exe|msi|apk|jar|md5|sig)(?:\?|$)/i ); @@ -29,6 +32,7 @@ describe('getConfig', () => { MATOMO_TIMEOUT_MS: '8000', LOG_LEVEL: 'debug', USER_AGENT_ALLOWLIST_REGEX: 'CustomBot', + URL_EXCLUDE_REGEX: '\\.(?:js|css)$', DOCUMENT_REGEX: '\\.custom$' }); expect(config).toMatchObject({ @@ -38,6 +42,7 @@ describe('getConfig', () => { logLevel: 'debug' }); expect(config.userAgentAllowlistRegex).toEqual(/CustomBot/i); + expect(config.urlExcludeRegex).toEqual(/\.(?:js|css)$/i); expect(config.documentRegex).toEqual(/\.custom$/i); }); @@ -45,6 +50,9 @@ describe('getConfig', () => { expect(() => getConfig({ ...baseEnv, USER_AGENT_ALLOWLIST_REGEX: '[' }) ).toThrow(/Invalid USER_AGENT_ALLOWLIST_REGEX/); + expect(() => getConfig({ ...baseEnv, URL_EXCLUDE_REGEX: '[' })).toThrow( + /Invalid URL_EXCLUDE_REGEX/ + ); expect(() => getConfig({ ...baseEnv, DOCUMENT_REGEX: '[' })).toThrow( /Invalid DOCUMENT_REGEX/ ); diff --git a/tests/matomo.test.ts b/tests/matomo.test.ts index a3f6f16..62e0622 100644 --- a/tests/matomo.test.ts +++ b/tests/matomo.test.ts @@ -77,6 +77,19 @@ describe('buildMatomoPayload (Worker)', () => { expect(payload).toBeNull(); }); + it('returns null when url is excluded', () => { + const response = new Response(null, { status: 200 }); + const payload = buildMatomoPayload( + new Request('https://example.com/assets/app.js?ver=1', { + headers: { 'user-agent': 'AgentX' } + }), + response, + 10, + config + ); + expect(payload).toBeNull(); + }); + it('uses defaults when user agent allowlist is disabled', () => { const response = new Response(null, { status: 200 }); const payload = buildMatomoPayload(