fix: implement cachableContentType, default to dynaic only

Author: ivictbor

README.md

@@ -17,6 +17,7 @@ It is designed for Next.js/Nuxt.js and other dynamic origins where latency and o
 - Built-in Basic-Auth dashboard for stats, charts, and invalidation
 - Warmup and sitemap discovery to reduce cold-start misses
 - Explicit response marker `X-Wait0` (`hit`, `miss`, `bypass`, etc.)
+- Diagnostic `X-Wait0-Reason` header when a response is not cacheable
 
 ## Quick Start
 
@@ -34,13 +35,16 @@ server:
   origin: 'http://localhost:8080'
 
 rules:
-  - match: PathPrefix(/)
+  - match: PathPrefix(/blog)
     priority: 1
+    varyByQueryParams: ['page']
+    # Stale-after signal: serve cached content, then refresh in background.
     expiration: '1m'
 
-  - match: PathPrefix(/blog)
+  - match: PathPrefix(/)
     priority: 2
-    varyByQueryParams: ['page']
+    # Defaults to HTML and XHTML, keeping static assets out of wait0.
+    cachableContentType: ['text/html', 'application/xhtml+xml']
     expiration: '1m'
 ```
 
@@ -129,29 +133,127 @@ Notes:
 
 | Guide | Description |
 |-------|-------------|
+| [Usage and configuration](#usage-and-configuration) | [How wait0 works](#how-wait0-works), [query parameters](#query-parameter-caching), [sitemap warmup](#sitemap-warmup), and the [config reference](#config-file-reference) |
 | [For Developers](docs/for-developers.md) | Build/test commands, config reference, runtime options |
 | [API Endpoints](docs/api-endpoints.md) | Proxy behavior, invalidation API, schemas, status codes |
 
-## Query-Aware Cache Keys
+## Usage and configuration
+
+### How wait0 works
+
+wait0 checks RAM, then disk, and waits for the origin only on a cache miss. A cached response is returned immediately. **Even when it is older than `expiration`, wait0 serves it first and refreshes it asynchronously.** Therefore, `expiration: '1m'` is a stale-after signal, not a hard expiry or eviction.
+
+wait0 caches only `GET` responses with a `2xx` status, an allowed `Content-Type`, and no `Cache-Control: no-cache` or `no-store` directive. `cachableContentType` defaults to `text/html` and `application/xhtml+xml`, including values with parameters such as `text/html; charset=utf-8`. This keeps wait0 focused on controllable dynamic SWR; static assets should normally be cached by a CDN, Nginx, and browser caching.
+
+An origin response that fails these checks is served as `X-Wait0: bypass` without being stored, with `X-Wait0-Reason` explaining why. If background revalidation receives a disallowed content type, either cache-control directive, or a non-`2xx` status, the existing entry is deleted; a network error leaves it available. A stale response is reported as `X-Wait0: hit`.
+
+### Query parameter caching
+
+Cache keys use only the URL path by default, so `/blog`, `/blog?page=1`, and `/blog?utm_source=email` share the same cached response. The first cold request reaches the origin with its complete query and fills the shared entry; later background refreshes omit query parameters that are not in `varyByQueryParams`.
+
+Use `varyByQueryParams` to explicitly choose the parameters that must create different entries:
+
+```yaml
+varyByQueryParams: ['page', 'lang']
+```
+
+With this setting, `/blog?page=1` and `/blog?page=2` are different cache entries, while an unlisted parameter such as `utm_source` is ignored for cache identity and background refreshes. Allowed parameter names and values are normalized into a stable order.
 
-Use `varyByQueryParams` on a rule when only a small set of query parameters should affect cache identity.
+### Sitemap warmup
 
-Example:
+`urlsDiscover` reads sitemap files and registers their paths in the disk cache. It supports sitemap indexes, gzip sitemaps, absolute URLs, and origin-relative paths. Discovery does not fetch each page body by itself: add `warmUp` to a matching rule to fetch and periodically refresh the discovered paths. Paths with no matching rule, or a rule with `bypass: true`, are ignored.
+
+### Config file reference
+
+This example contains every current configuration option. Durations use Go syntax such as `10s`, `1m`, or `2h`; sizes accept bytes or `k`, `m`, and `g` suffixes.
 
 ```yaml
+storage:
+  ram:
+    # Maximum RAM cache size.
+    max: '100m'
+  disk:
+    # Maximum LevelDB cache size.
+    max: '1g'
+
+server:
+  # HTTP port; defaults to 8080 when omitted or set to 0.
+  port: 8082
+  # Required origin base URL.
+  origin: 'http://host.docker.internal:8080'
+
+  invalidation:
+    # Enables POST /wait0/invalidate.
+    enabled: true
+    # Maximum number of waiting asynchronous invalidation jobs; default 128.
+    queue_size: 128
+    # Number of invalidation workers; default 4.
+    worker_concurrency: 4
+    # Maximum invalidation request body in bytes; default 1 MiB.
+    max_body_bytes: 1048576
+    # Recommended maximum normalized paths per request; default 1024.
+    max_paths_per_request: 1024
+    # Recommended maximum normalized tags per request; default 1024.
+    max_tags_per_request: 1024
+    # true rejects requests over path/tag limits; false accepts and logs them.
+    hard_limits: false
+
+auth:
+  tokens:
+    # Unique name used for authentication and audit logs.
+    - id: 'backoffice'
+      # Optional inline secret/fallback; prefer token_env in production.
+      token: 'replace-me'
+      # Environment variable whose non-empty value overrides token.
+      token_env: 'WAIT0_API_AUTH_TOKEN'
+      # invalidation:write protects invalidation; stats:read protects stats.
+      scopes: ['invalidation:write', 'stats:read']
+
+urlsDiscover:
+  # Delay before the first sitemap scan; 0s scans immediately.
+  initialDelay: '20s'
+  # Rescan interval; omit to scan only once.
+  rediscoverEvery: '10m'
+  # Sitemap or sitemap-index URLs; origin-relative paths are also accepted.
+  sitemaps:
+    - 'https://example.com/sitemap.xml'
+
 rules:
-  - match: PathPrefix(/blog)
+  # Matches path prefixes; combine alternatives with |.
+  - match: PathPrefix(/api) | PathPrefix(/admin)
+    # Lower priority is evaluated first; the first matching rule wins.
     priority: 1
-    varyByQueryParams: ['page']
+    # Skips cache lookup and storage for matching requests.
+    bypass: true
+
+  - match: PathPrefix(/)
+    priority: 2
+    # Bypasses cache when any named cookie is present.
+    bypassWhenCookies: ['sessionid']
+    # Exact media types eligible for wait0 caching. Parameters such as charset
+    # are ignored. Defaults to the two values shown here.
+    cachableContentType: ['text/html', 'application/xhtml+xml']
+    # Only these query parameters become part of the cache key.
+    varyByQueryParams: ['page', 'lang']
+    # Stale-after age: serve cached data immediately, then refresh async;
+    # omit to disable request-triggered age refresh.
     expiration: '1m'
+    warmUp:
+      # Refresh all known matching paths at this interval, regardless of age.
+      runEvery: '10m'
+      # Maximum concurrent requests for this warmup rule.
+      maxRequestsAtATime: 20
+
+logging:
+  # Logs a stats snapshot at this interval; omit to disable.
+  log_stats_every: '1m'
+  # Logs a summary after each warmup batch.
+  log_warmup: true
+  # Logs details for each scanned sitemap.
+  log_url_autodiscover: true
 ```
 
-With that rule:
-
-- `/blog` and `/blog?utm_source=newsletter` use the same cache entry because `utm_source` is ignored.
-- `/blog?page` uses a different cache entry from `/blog`.
-- `/blog?page=1` uses a different cache entry from both `/blog` and `/blog?page`.
-- If multiple values are present for an allowed parameter, wait0 canonicalizes them into a stable key order.
+Compatibility-only options are still accepted but should not be used in new files: `urlsDiscover.initalDelay` is the old spelling of `initialDelay`, `logging.log_revalidation_every` is replaced by `log_warmup`, and `server.invalidation.tokens` is replaced by top-level `auth.tokens`.
 
 ## Redeploy Note
 
@@ -163,7 +265,7 @@ If stale HTML is still cached, clients can receive pages that point to missing a
 To reduce this risk, `wait0` clears disk cache on startup by default (`WAIT0_INVALIDATE_DISK_CACHE_ON_START=true`).
 That behavior makes rollout safer because old HTML is not reused across deploy generations.
 
-Hpowever, during redeploy you still need to restart wait0 by explicitly orchestrating it (for example,  docker restart wait0).
+However, during redeploy you still need to restart wait0 by explicitly orchestrating it (for example, `docker restart wait0`).
 
 If you do not restart between deploys, proactively refresh cache using invalidation and warmup for critical routes.
 
@@ -174,7 +276,8 @@ If you do not restart between deploys, proactively refresh cache using invalidat
 - Rule field `varyByQueryParams[]` opts specific query parameters into cache identity for matching paths.
 - Query parameters not listed in `varyByQueryParams[]` and all fragments are ignored for cache identity.
 - Only `GET` requests are cache candidates.
-- Only origin `2xx` responses are stored.
-- For stale entries (rule `expiration`), wait0 serves cached response immediately and revalidates asynchronously.
-- For origin non-`2xx`, wait0 skips caching and evicts any existing key for that path.
+- Only origin `2xx` responses matching the rule's `cachableContentType` allowlist are stored.
+- `cachableContentType` defaults to HTML and XHTML; use CDN, reverse-proxy, and browser caches for static assets.
+- Rule `expiration` marks entries stale but does not evict them; stale responses are served immediately and revalidation is scheduled on a best-effort basis.
+- On a cache-path miss or revalidation, an origin non-`2xx` is not cached and any existing key is evicted.
 - Invalidation is asynchronous: accept request -> resolve keys by `paths` and `tags` -> delete keys -> recrawl in background. Path invalidation clears all cached query-aware variants for that path.

docs/api-endpoints.md

@@ -28,28 +28,34 @@ This is the default request path handled by the proxy controller.
 
 ## Behavior
 
-| Condition | Result | `X-Wait0` |
-|----------|--------|-----------|
-| Matching rule has `bypass: true` | Forward to origin, no cache write | `bypass` |
-| Matching rule cookie bypass is triggered | Forward to origin, no cache write | `ignore-by-cookie` |
-| Method is not `GET` | Forward to origin, no cache write | `bypass` |
-| RAM or disk hit for active entry | Serve cached response instantly | `hit` |
-| Miss and cacheable origin `2xx` | Store and serve response | `miss` |
-| Origin non-`2xx` | Do not cache, evict existing key | `ignore-by-status` |
-| Origin fetch/network failure | Gateway error | `bad-gateway` |
+| Condition | Result | `X-Wait0` | `X-Wait0-Reason` |
+|----------|--------|-----------|------------------|
+| Matching rule has `bypass: true` | Skip cache; fetch upstream with bodyless `GET` | `bypass` | `bypass-rule` |
+| Matching rule cookie bypass is triggered | Skip cache; fetch upstream with bodyless `GET` | `ignore-by-cookie` | `bypass-cookie` |
+| Method is not `GET` | Skip cache; convert to bodyless upstream `GET` | `bypass` | `non-get-method` |
+| RAM or disk hit for active entry | Serve cached response instantly | `hit` | absent |
+| Miss and cacheable origin `2xx` | Store and serve response | `miss` | absent |
+| Origin `2xx` has disallowed `Content-Type` | Serve without storing | `bypass` | `non-cacheable-content-type` |
+| Origin `2xx` has `no-cache` or `no-store` | Serve without storing | `bypass` | `non-cacheable-cache-control` |
+| Origin non-`2xx` | Do not cache, evict existing key | `ignore-by-status` | `non-cacheable-status` |
+| Origin fetch/network failure | Gateway error | `bad-gateway` | `origin-error` |
 
 ## Cacheability rule
 
 An origin response is cacheable only when:
 
 - status is `2xx`, and
+- the media type in `Content-Type` appears in the matching rule's `cachableContentType` list, and
 - `Cache-Control` does not include `no-store` or `no-cache`.
 
+`cachableContentType` defaults to `text/html` and `application/xhtml+xml`. Matching is case-insensitive and ignores media-type parameters such as `charset=utf-8`. A missing or malformed `Content-Type` is not cacheable.
+
 ## Response headers added by wait0
 
 | Header | When present | Meaning |
 |--------|--------------|---------|
 | `X-Wait0` | always on handled responses | Cache/proxy decision marker |
+| `X-Wait0-Reason` | response bypassed or failed | Machine-readable reason the response was not cached |
 | `X-Wait0-Revalidated-At` | cache `hit` with revalidation metadata | Last revalidation timestamp (RFC3339Nano) |
 | `X-Wait0-Revalidated-By` | with `X-Wait0-Revalidated-At` | Revalidation source (`user`, `warmup`, `invalidate`, etc.) |
 | `X-Wait0-Discovered-By` | if entry was discovery seeded | Discovery source marker |

docs/for-developers.md

@@ -128,6 +128,7 @@ For dashboard:
 | `priority` | no | Rules are sorted ascending by priority |
 | `bypass` | no | For matching paths, bypass cache completely |
 | `bypassWhenCookies[]` | no | If any listed cookie exists, bypass cache |
+| `cachableContentType[]` | no | Exact cache-eligible media types; defaults to `text/html` and `application/xhtml+xml`; parameters are ignored |
 | `varyByQueryParams[]` | no | Query params that should participate in cache identity for matching paths |
 | `expiration` | no | Duration for stale check and async revalidation |
 | `warmUp.runEvery` | with `warmUp` | Duration, must be `> 0` |
@@ -157,9 +158,13 @@ For dashboard:
 - Rule field `varyByQueryParams[]` opt-ins selected query params so `/a/b?page`, `/a/b?page=1`, and `/a/b` can be distinct cache entries.
 - Query params not listed in `varyByQueryParams[]` and all fragments are ignored for cache identity.
 - Only `GET` requests are cache-eligible.
+- Bypassed and non-`GET` requests are sent upstream as `GET` without the original body.
 - Non-2xx origin responses are not cached and existing cached key is removed.
-- Dynamic pages are expected to send `Cache-Control: no-cache` or `no-store` so wait0 treats them as passthrough and revalidation-managed.
+- Origin responses whose media type is not in `cachableContentType` are served but not stored; background revalidation deletes an existing entry if its new media type is disallowed.
+- We recommend to Keep static assets in CDN, Nginx, and browser caches; wait0 defaults to caching only dynamic HTML/XHTML SWR responses.
+- Origin `2xx` responses with `Cache-Control: no-cache` or `no-store` are not stored; either directive received during revalidation deletes the existing entry.
 - `X-Wait0` response header identifies behavior (`hit`, `miss`, `bypass`, `ignore-by-cookie`, `ignore-by-status`, `bad-gateway`).
+- `X-Wait0-Reason` identifies why a response was bypassed or failed (for example, `bypass-rule` or `non-cacheable-content-type`).
 
 ## See Also
 

internal/wait0/config.go

@@ -100,13 +100,14 @@ type WarmUpConfig struct {
 }
 
 type Rule struct {
-	Match             string        `yaml:"match"`
-	Priority          int           `yaml:"priority"`
-	Bypass            bool          `yaml:"bypass"`
-	BypassWhenCookies []string      `yaml:"bypassWhenCookies"`
-	VaryByQueryParams []string      `yaml:"varyByQueryParams"`
-	Expiration        string        `yaml:"expiration"`
-	WarmUp            *WarmUpConfig `yaml:"warmUp"`
+	Match                string        `yaml:"match"`
+	Priority             int           `yaml:"priority"`
+	Bypass               bool          `yaml:"bypass"`
+	BypassWhenCookies    []string      `yaml:"bypassWhenCookies"`
+	CachableContentTypes []string      `yaml:"cachableContentType"`
+	VaryByQueryParams    []string      `yaml:"varyByQueryParams"`
+	Expiration           string        `yaml:"expiration"`
+	WarmUp               *WarmUpConfig `yaml:"warmUp"`
 
 	// compiled
 	matchers  []pathPrefixMatcher
@@ -205,6 +206,11 @@ func LoadConfig(path string) (Config, error) {
 			return Config{}, fmt.Errorf("rules[%d].match: %w", i, err)
 		}
 		r.matchers = ms
+		contentTypes, err := proxy.NormalizeCachableContentTypes(r.CachableContentTypes)
+		if err != nil {
+			return Config{}, fmt.Errorf("rules[%d].cachableContentType: %w", i, err)
+		}
+		r.CachableContentTypes = contentTypes
 		if len(r.VaryByQueryParams) > 0 {
 			params, err := proxy.NormalizeVaryByQueryParams(r.VaryByQueryParams)
 			if err != nil {

internal/wait0/config_test.go

@@ -32,7 +32,8 @@ rules:
     bypass: true
   - match: "PathPrefix(/)"
     priority: 1
-		varyByQueryParams: [" page ", "lang", "page"]
+    cachableContentType: [" Application/JSON; Charset=UTF-8 ", "application/json"]
+    varyByQueryParams: [" page ", "lang", "page"]
     expiration: "30s"
     warmUp:
       runEvery: "1m"
@@ -77,6 +78,18 @@ rules:
 			t.Fatalf("varyByQueryParams[%d] = %q, want %q", i, cfg.Rules[0].VaryByQueryParams[i], wantQueryParams[i])
 		}
 	}
+	if got := cfg.Rules[0].CachableContentTypes; len(got) != 1 || got[0] != "application/json" {
+		t.Fatalf("cachableContentType = %v, want [application/json]", got)
+	}
+	wantDefaultContentTypes := []string{"text/html", "application/xhtml+xml"}
+	if got := cfg.Rules[1].CachableContentTypes; len(got) != len(wantDefaultContentTypes) {
+		t.Fatalf("default cachableContentType = %v, want %v", got, wantDefaultContentTypes)
+	}
+	for i, want := range wantDefaultContentTypes {
+		if got := cfg.Rules[1].CachableContentTypes[i]; got != want {
+			t.Fatalf("default cachableContentType[%d] = %q, want %q", i, got, want)
+		}
+	}
 	if cfg.Rules[0].warmEvery != time.Minute || cfg.Rules[0].warmMax != 3 {
 		t.Fatalf("warmup compiled fields not set")
 	}
@@ -90,6 +103,7 @@ func TestLoadConfig_Errors(t *testing.T) {
 		{name: "missing origin", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  port: 8080\nrules: []\n"},
 		{name: "bad match", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\nrules:\n  - match: \"BadExpr(/)\"\n"},
 		{name: "bad varyByQueryParams", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\nrules:\n  - match: \"PathPrefix(/)\"\n    varyByQueryParams: [\"page\", \" \" ]\n"},
+		{name: "bad cachableContentType", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\nrules:\n  - match: \"PathPrefix(/)\"\n    cachableContentType: [\"not a content type\"]\n"},
 		{name: "bad warmup", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\nrules:\n  - match: \"PathPrefix(/)\"\n    warmUp:\n      runEvery: \"\"\n      maxRequestsAtATime: 1\n"},
 		{name: "bad log stats", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\nlogging:\n  log_stats_every: \"bad\"\nrules: []\n"},
 		{name: "duplicate auth token ids", yaml: "storage:\n  ram: {max: \"1m\"}\n  disk: {max: \"1m\"}\nserver:\n  origin: \"http://x\"\n  invalidation:\n    enabled: true\nauth:\n  tokens:\n    - id: \"dup\"\n      token: \"a\"\n      scopes: [\"invalidation:write\"]\n    - id: \"dup\"\n      token: \"b\"\n      scopes: [\"invalidation:write\"]\nrules: []\n"},