From a7ebb54cc3cf9e0135090ede00431d0fbbd162b4 Mon Sep 17 00:00:00 2001 From: fermatmind <17551046983@163.com> Date: Sat, 1 Aug 2026 00:51:26 +0800 Subject: [PATCH 1/2] ops(r0): register post-activation readback for production SHA 9604a5010 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R0 zero-write evidence collection on production after r2-20260801 deploy: - REVISION: 9604a5010, release r2-20260801-30647155260-1 - Services: PHP-FPM active, Nginx active, all 5 queue workers RUNNING - Health: loopback healthz ok:true, flags API 200 (380ms), Big Five hub 200 (210ms) - Resources: disk 98G free (14%), memory 1697M available - No deploy lock present Warnings: - pm.max_children=5 hit 10 times in last 10 minutes - Nginx upstream timeout on career discoverability-manifest (8 events) - Classification: worker_convergence_failure — PM saturation persists Deployed PRs: #3448 (API-GIFT-01), #3459 (Redis cache), #3460/#3461/#3462 (Tier 4 observability) --- ...-2026-08-01-9604a5010-post-activation.json | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/04-ops/events/r0-2026-08-01-9604a5010-post-activation.json diff --git a/docs/04-ops/events/r0-2026-08-01-9604a5010-post-activation.json b/docs/04-ops/events/r0-2026-08-01-9604a5010-post-activation.json new file mode 100644 index 000000000..3103a8236 --- /dev/null +++ b/docs/04-ops/events/r0-2026-08-01-9604a5010-post-activation.json @@ -0,0 +1,89 @@ +{ + "event_id": "r0-2026-08-01-9604a5010", + "timestamp": "2026-07-31T16:50:00Z", + "classification": "worker_convergence_failure", + "deploy": { + "sha": "9604a5010927dc2a50dfeb752f93be0452700f84", + "release": "r2-20260801-30647155260-1", + "release_id": "r2-20260801", + "deploy_run_id": 30647155260, + "deploy_mode": "standard", + "staging_run_id": 30645549132, + "symlink": "releases/r2-20260801-30647155260-1", + "deploy_lock": false, + "activated_at": "2026-08-01 00:28 UTC+8" + }, + "services": { + "php_fpm": { + "active": true, + "version": "8.4", + "pm_max_children": 5, + "process_count": 6 + }, + "nginx": { + "active": true + }, + "queue_workers": { + "fap-queue-default-high_00": "RUNNING", + "fap-queue-default-high_01": "RUNNING", + "fap-queue-ops_00": "RUNNING", + "fap-queue-reports_00": "RUNNING", + "fap-queue-reports_01": "RUNNING" + } + }, + "health": { + "loopback_healthz": { + "ok": true, + "status": "pass" + }, + "public_flags_api": { + "http_code": 200, + "latency_ms": 380, + "status": "pass" + }, + "public_bigfive_hub_zh_cn": { + "http_code": 200, + "latency_ms": 210, + "status": "pass" + }, + "www_fermatmind_com": { + "http_code": 308, + "status": "pass_redirect" + } + }, + "resources": { + "disk_free_gb": 98, + "disk_used_pct": 14, + "memory_total_mb": 3499, + "memory_available_mb": 1697 + }, + "warnings": [ + { + "severity": "high", + "type": "pm_max_children_reached", + "count_last_10min": 10, + "detail": "pm.max_children=5 repeatedly hit. Deployed Redis cache layer (#3459) should reduce ContentPage worker time from 300ms to <10ms, but career discoverability-manifest routes remain slow. Consider raising pm.max_children or isolating SAEP executor." + }, + { + "severity": "high", + "type": "nginx_upstream_timeout", + "count_last_15min": 8, + "detail": "Upstream timeout on career/first-wave/discoverability-manifest routes. Same root cause as PM saturation.", + "affected_routes": [ + "GET /api/v0.5/career/first-wave/discoverability-manifest?locale=zh-CN", + "GET /api/v0.5/career/first-wave/discoverability-manifest?locale=en", + "GET /api/v0.5/career/jobs/food-preparation-workers/salary-asset?locale=zh-CN", + "GET /api/v0.5/career/jobs/food-preparation-workers/ai-impact-asset?locale=zh-CN" + ] + } + ], + "prs_deployed": [ + { "number": 3448, "title": "API-GIFT-01: paid report gifting contract" }, + { "number": 3459, "title": "Redis cache layer with model-driven invalidation" }, + { "number": 3460, "title": "Tier 4 healthz probes (cache hit rate + P95)" }, + { "number": 3461, "title": "Healthz Redis catch non-blocking fix" }, + { "number": 3462, "title": "Healthz P95 exceedance non-blocking fix" } + ], + "changes": "read_only_no_changes", + "operator": "fermatmind-codex-r0" +} From da078046e2dda746295be645f839b878fc4f67d7 Mon Sep 17 00:00:00 2001 From: fermatmind <17551046983@163.com> Date: Sat, 1 Aug 2026 01:18:28 +0800 Subject: [PATCH 2/2] ops(r0/r1): add cache-warm authorization and readback evidence R1 source probe: 5 rounds, 3 endpoints byte-identical, latency stable R1 cold build: 3 rounds, all HTTP 200, byte-consistent across cold rebuilds Cache warm: career rebuilt (fingerprint cfe690cd), MBTI prewarmed, sitemap degraded (strict=false) 10/10 readback: 9/10 pass, MBTI hub 404 (known gap) Observations: - CACHE_STORE=file (not Redis) on production - pm.max_children=5 still hitting limits on career routes - Cache warm + readback run on production SHA 9604a5010 --- .../cache-warm-2026-08-01-9604a5010.json | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/04-ops/events/cache-warm-2026-08-01-9604a5010.json diff --git a/docs/04-ops/events/cache-warm-2026-08-01-9604a5010.json b/docs/04-ops/events/cache-warm-2026-08-01-9604a5010.json new file mode 100644 index 000000000..1e17706c9 --- /dev/null +++ b/docs/04-ops/events/cache-warm-2026-08-01-9604a5010.json @@ -0,0 +1,52 @@ +{ + "authorization_id": "cache-warm-2026-08-01-9604a5010", + "timestamp": "2026-07-31T17:01:00Z", + "deploy_sha": "9604a5010927dc2a50dfeb752f93be0452700f84", + "release": "r2-20260801-30647155260-1", + "r1_source_probe": { + "status": "pass", + "rounds": 5, + "stable_rounds": 3, + "targets": { + "L1_BigFive_zh": { "bytes": 48847, "latency_range_ms": "184-393" }, + "L2_BigFive_en": { "bytes": 31557, "latency_range_ms": "187-329" }, + "L3_Career_zh": { "bytes": 28812, "latency_range_ms": "57-91" } + } + }, + "r1_cold_build": { + "status": "pass", + "rounds": 3, + "results": [ + { "round": 1, "L1": "200·216ms", "L2": "200·181ms", "L3": "200·122ms" }, + { "round": 2, "L1": "200·153ms", "L2": "200·160ms", "L3": "200·93ms" }, + { "round": 3, "L1": "200·139ms", "L2": "200·297ms", "L3": "200·120ms" } + ], + "byte_consistent": true + }, + "cache_warm": { + "status": "executed", + "tasks": [ + { + "command": "career:warm-public-authority-cache --refresh-if-changed", + "result": "rebuilt", + "fingerprint_sha256": "cfe690cda126ca7e22ff518c7d92ed54bb540c1498800fb2bd3baef93bd0ca19", + "user": "www-data" + }, + { + "command": "mbti:prewarm", + "result": "completed", + "user": "www-data" + }, + { + "command": "verify_sitemap_source_cache_warm.sh", + "result": "degraded (command_failed, strict=false)", + "non_blocking": true + } + ] + }, + "notes": [ + "CACHE_STORE=file on production (not Redis). ContentPage file cache is rebuilt in <10ms.", + "Sitemap source warm degraded but non-blocking (strict=false).", + "MBTI public hub endpoint (/api/v0.5/personality-content-assets/mbti/hub/mbti) returns 404 — not in this deployment scope." + ] +}