Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
114 commits
Select commit Hold shift + click to select a range
c4b7f09
fix(stateless): contain watch loss and preserve desktop chats
alfredolopez80 Sep 24, 2026
b3aae3b
test(control-api): pin refresh rate-limit window
alfredolopez80 Sep 24, 2026
e4d4dc3
Merge branch 'dev' into fix/stateless-incident-containment
alfredolopez80 Sep 25, 2026
f7349f4
fix(desktop,hcc): fence destructive scope, distinguish host authority…
alfredolopez80 Sep 25, 2026
7b77182
fix(hcc,desktop): contain scoped deletions and truthful runtime holds
alfredolopez80 Sep 25, 2026
cd97555
Merge branch 'dev' into fix/stateless-incident-containment
alfredolopez80 Sep 25, 2026
d69d212
test(hcc): type the legacy deployment fence fixture
alfredolopez80 Sep 25, 2026
d99a0b3
test(desktop): update host authority fixtures
alfredolopez80 Sep 25, 2026
b70a66e
fix(desktop,hcc): harden revoked reads and draining holds
alfredolopez80 Sep 25, 2026
1e15b4c
Merge remote-tracking branch 'origin/dev' into fix/stateless-incident…
alfredolopez80 Sep 25, 2026
8d669ad
test(desktop): record two-host stateless continuity journey
alfredolopez80 Sep 25, 2026
b59ff67
test(stateless): align recorded journey with owned runtime
alfredolopez80 Sep 25, 2026
a7d2e5d
fix(desktop,hcc): preserve pending stateless conversations
alfredolopez80 Sep 25, 2026
572675c
test(desktop): prove stateless controlled parity
alfredolopez80 Sep 25, 2026
e689b49
test(desktop): wake stateless baseline through catalog
alfredolopez80 Sep 25, 2026
3d1b7cf
test(desktop): align controlled GFS prompt
alfredolopez80 Sep 25, 2026
9319fe4
test(desktop): verify stateless pending shell work
alfredolopez80 Sep 25, 2026
8da016b
test(desktop): make controlled gfs read deterministic
alfredolopez80 Sep 25, 2026
a76743e
fix(mcp-host): preserve zai reasoning across tools
alfredolopez80 Sep 26, 2026
e630251
test(desktop): harden controlled parity journey
alfredolopez80 Sep 26, 2026
5a7185f
fix(mcp-host): avoid shell workflow recovery drift
alfredolopez80 Sep 26, 2026
7d937bf
test(desktop): prove stateless host parity
alfredolopez80 Sep 26, 2026
fb834fc
fix(desktop): fence stateless prewarm across auth changes
alfredolopez80 Sep 26, 2026
df8f113
test(desktop): cover prewarm races and pending acknowledgement
alfredolopez80 Sep 26, 2026
17a0bab
test(proxy): characterize active wake admission gap
alfredolopez80 Sep 26, 2026
52a0049
test(e2e): measure stateless send-before-ready baseline
alfredolopez80 Sep 26, 2026
3233833
chore(dev): add autoresearch hard gates
alfredolopez80 Sep 26, 2026
24f0819
docs(minikube): require consent for recertification
alfredolopez80 Sep 26, 2026
38cd312
fix(harness): require node 24 for canonical gates
alfredolopez80 Sep 26, 2026
922ec30
test(desktop): harden human stateless journey close
alfredolopez80 Sep 26, 2026
cda0e9d
test(desktop): anchor stateless wake measurements
alfredolopez80 Sep 26, 2026
d188879
test(desktop): fix wake transcript anchor
alfredolopez80 Sep 26, 2026
bf1adff
fix(e2e): resolve wake recovery credential canonically
alfredolopez80 Sep 26, 2026
fa71a8c
fix(e2e): resolve sqlite reader in runtime layout
alfredolopez80 Sep 26, 2026
f09828f
test(e2e): measure prewarm readiness without censoring
alfredolopez80 Sep 26, 2026
705638b
test(e2e): attribute cold pod wake timeline
alfredolopez80 Sep 26, 2026
2c8a2a0
fix(e2e): create recovery artifact directory
alfredolopez80 Sep 26, 2026
d09578f
fix(proxy): admit messages during stateless wake
alfredolopez80 Sep 26, 2026
f508214
perf(mcp-host): lazy-load document tool libraries
alfredolopez80 Sep 26, 2026
297839a
fix(proxy): sanitize hostRef in wake logs
alfredolopez80 Sep 27, 2026
9116bd7
feat(control-ui): enable stateless agent creation
alfredolopez80 Sep 27, 2026
b86f873
fix(review): address proxy probe, desktop privacy, HCC timer, and wor…
alfredolopez80 Sep 27, 2026
53040e0
fix(review): remove mutating probe, fence HCC poll, and address all r…
alfredolopez80 Sep 27, 2026
c9f70ca
fix(harness): auto-clean dead port-forward PIDs and make tests locale…
alfredolopez80 Sep 27, 2026
293be7a
fix(codeql): remove unused variable and bound slug regex
alfredolopez80 Sep 27, 2026
a1f93cd
fix(codeql): deduplicate character class
alfredolopez80 Sep 27, 2026
f4ff4de
chore(proxy): remove unused isWakeCapable import
alfredolopez80 Sep 27, 2026
378d74f
fix(review): address zach88 and jozer-ami blockers and cheap lows
alfredolopez80 Sep 28, 2026
3076e85
fix(review): address Astra audit findings on scopes, timeouts, and ac…
alfredolopez80 Sep 28, 2026
70d53f2
fix(desktop): gate host blocking to auth errors only in catalog catches
alfredolopez80 Sep 28, 2026
2c2f360
fix(desktop): suppress offline toast for auth-blocked catalog errors
alfredolopez80 Sep 28, 2026
3540216
Merge remote-tracking branch 'origin/dev' into fix/stateless-incident…
alfredolopez80 Sep 28, 2026
b65165c
Merge remote-tracking branch 'origin/dev' into fix/stateless-incident…
alfredolopez80 Sep 28, 2026
7655ab3
fix(hcc): hold stateless runtimes with verified scopes
alfredolopez80 Sep 28, 2026
fdba1c5
fix(proxy): preserve bounded artifact transfers
alfredolopez80 Sep 28, 2026
9354cf8
fix(desktop): classify catalog failures across IPC
alfredolopez80 Sep 28, 2026
185c16c
test(hcc): satisfy full typescript check
alfredolopez80 Sep 28, 2026
02600fe
fix(hcc): bind preserved wake bootstraps
alfredolopez80 Sep 28, 2026
0a2ce67
fix(desktop): preserve cursor during rate limits
alfredolopez80 Sep 28, 2026
90f8b70
test(hcc): enforce held runtime process evidence
alfredolopez80 Sep 28, 2026
b1f72a7
fix(desktop): protect deleted chats and opt-in journeys
alfredolopez80 Sep 28, 2026
6e3da76
fix(mcp-host): recover quoted workflow intents
alfredolopez80 Sep 28, 2026
acf817d
test(rpc-proxy): prove wake timeout producers
alfredolopez80 Sep 28, 2026
d74204f
fix(control-ui): verify stateless host creation
alfredolopez80 Sep 28, 2026
48b2bed
fix(hcc): simplify preserved wake provisioning
alfredolopez80 Sep 28, 2026
cf56d61
test(t1): expose failed postgres suites
alfredolopez80 Sep 28, 2026
723b272
test(control-api): bound stress-test sockets below the macOS accept q…
alfredolopez80 Sep 28, 2026
4250386
docs(t2): record macos stress socket failures
alfredolopez80 Sep 28, 2026
a696391
test(control-api): make the depth fixture sanity check platform-indep…
alfredolopez80 Sep 25, 2026
724f818
fix(hcc): guard held-runtime credential renewal and bind wake bootstr…
alfredolopez80 Sep 29, 2026
81820e6
fix(rpc-proxy): mark revoked host access with a code and bound wake h…
alfredolopez80 Sep 29, 2026
72ff9f8
fix(control-api): send the host-access denial reason as a header
alfredolopez80 Sep 29, 2026
e0c9021
fix(mcp-host): narrow workflow intent recovery to the dev-accepted va…
alfredolopez80 Sep 29, 2026
c5c82c8
test(t1): report failed file messages and redact escaped DSNs in the …
alfredolopez80 Sep 29, 2026
54c4919
fix(desktop-ui): protect the chat catalog during host authority loss
alfredolopez80 Sep 29, 2026
2c6befe
fix(desktop): classify upstream errors and keep deleted chats deleted
alfredolopez80 Sep 29, 2026
110d746
fix(t1): assemble synthetic DSNs at run time in the reporter test
alfredolopez80 Sep 29, 2026
ce53e25
fix(rpc-proxy): keep mcp-host 403 codes reserved and stop replaying m…
alfredolopez80 Sep 29, 2026
6df1be4
fix(hcc): fail closed on a stale McpServer cache and hold a running s…
alfredolopez80 Sep 29, 2026
13f5aa0
fix(desktop): rebind the chat store after a failed team restore and p…
alfredolopez80 Sep 29, 2026
f3d7097
fix(t1): redact more password spellings and keep the reporter JSON valid
alfredolopez80 Sep 29, 2026
98e8837
Merge remote-tracking branch 'origin/dev' into fix/stateless-incident…
alfredolopez80 Sep 29, 2026
cc111db
fix(profile-ui): track the production next-env.d.ts so T0 leaves the …
alfredolopez80 Sep 29, 2026
65bc16c
Merge remote-tracking branch 'origin/dev' into fix/stateless-incident…
alfredolopez80 Sep 29, 2026
aafb502
test(hcc): derive the runtime credential fixture from a real mint pass
alfredolopez80 Sep 29, 2026
7eed4e7
fix(hcc): consume a fresh bootstrap on GFS-only refresh when the depl…
alfredolopez80 Sep 29, 2026
2327789
fix(hcc): defer instead of downgrading oauth:user-token when the McpS…
alfredolopez80 Sep 29, 2026
98685c9
fix(hcc): keep a Ready runtime deployed when the held GFS renewal fails
alfredolopez80 Sep 29, 2026
e073d00
test(hcc): prove the draining revert requires a stateless fresh read
alfredolopez80 Sep 29, 2026
a9a11cd
fix(rpc-proxy): retry idempotent GETs until the wake-hold deadline
alfredolopez80 Sep 29, 2026
51ea9a1
fix(rpc-proxy): pin the wake-hold deadline before host resolution
alfredolopez80 Sep 29, 2026
ab92407
fix(rpc-proxy): abort mutating upstream calls when the client disconn…
alfredolopez80 Sep 29, 2026
cd8e115
fix(rpc-proxy): subject_mismatch is a denial, not a revocation
alfredolopez80 Sep 29, 2026
ba86433
test(rpc-proxy): guard the host-access denial contract across services
alfredolopez80 Sep 29, 2026
0432fdb
fix(desktop): write the tombstone sidecar before the chat index
alfredolopez80 Sep 29, 2026
df32860
test(desktop): cover sidecar read failures, shape validation and prec…
alfredolopez80 Sep 29, 2026
a8e3698
fix(desktop): drop over-long session cursors and stop treating 408 as…
alfredolopez80 Sep 29, 2026
30ecd0d
fix(desktop-ui): invalidate the session catalog cache when a Host is …
alfredolopez80 Sep 29, 2026
777278c
test(desktop-ui): use IPC-shaped 403 fixtures with same-scope controls
alfredolopez80 Sep 29, 2026
62f6968
fix(desktop-ui): remove the unreachable rename alternative from the r…
alfredolopez80 Sep 29, 2026
f48785b
test(desktop-ui): delegate the chat fixture to the real ChatStore
alfredolopez80 Sep 30, 2026
48e3b7f
fix(desktop): sweep pending chat cleanups per agent on read and once …
alfredolopez80 Sep 29, 2026
07c16b0
fix(mcp-host): accept trailing comma, semicolon and colon on direct r…
alfredolopez80 Sep 29, 2026
15dd1ba
fix(scripts): redact non-ASCII passwords in both JSON escape spellings
alfredolopez80 Sep 29, 2026
c6690ea
ci: run the autoresearch gate self-test in shell-syntax
alfredolopez80 Sep 29, 2026
0866486
fix(scripts): close the preflight phase in certification evidence
alfredolopez80 Sep 29, 2026
7e56e5f
fix(mcp-host): reject an invalid sessions cursor in the session list …
alfredolopez80 Sep 29, 2026
9f4d347
test(hcc): type the renewed GFS mint mock as a GfsHostToken
alfredolopez80 Sep 30, 2026
6ec74d7
fix(hcc): report a held renewal failure as telemetry, not as a failed…
alfredolopez80 Sep 30, 2026
f0dab25
fix(rpc-proxy): refuse a task cancel without the caller subject
alfredolopez80 Sep 30, 2026
2279431
fix(rpc-proxy): abort a mutating call when the client left during hos…
alfredolopez80 Sep 30, 2026
b29077d
fix(desktop-ui): keep the newer catalog request when an invalidated o…
alfredolopez80 Sep 30, 2026
70b7ef4
fix(rpc-proxy): release the session rename upstream call when the cli…
alfredolopez80 Sep 30, 2026
051cefa
test(desktop-ui): prove catalog merge composes over disjoint pages
alfredolopez80 Sep 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .cursor/skills/minikube-t0-t1-t2/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,18 @@ Before running anything, verify ALL of these:
`pre-gate-sync` can restart every deployment; T2 then renews the hold
once, before Health/Playwright, and records `PortForwards=` evidence.
Without it an in-run sync leaves the hold on terminated pods.
- [ ] When the same owned profile is reused after a HEAD change, the
orchestrator runs `pre-gate-sync` for a targeted change and refreshes
forwards after any restarted services. For `targeted-sync`, also supply
a bounded `T2_HEALTHCHECK_COMMAND` that exercises the affected
user-facing path.
- [ ] Before a login-based health journey patches HCC or creates fault
fixtures, resolve its seeded test credential in-process. Source
`scripts/e2e/load-dotenv.sh` and `scripts/e2e/admin-credentials.sh`;
use `dotenv_load_canonical_root` and `e2e_resolve_admin_password`.
An explicit `E2E_USER_PASSWORD` may override the journey credential.
Never inspect, print, log, or persist `.env` contents, even for a
key-presence check. Fail before cluster mutation if resolution fails.

Shell contract-test rule: fixtures that exercise Git/lease state must use a
temporary repository via `scripts/tests/lib/minikube-fixture-repo.sh`. Keep the
Expand Down Expand Up @@ -113,6 +125,12 @@ Rules that override any shortcut idea:
- Never set `T2_RUN_T0=false` / `T2_RUN_T1=false` by hand to skip a lane that
was not certified on this HEAD; the harness refuses it unless the plan is
`already-synced`, and evidence must show the earlier green runs.
- Before repeating a completed full T0/T1/T2 certification, propose the
repeat with its reason and expected cost and ask the user to decide. A
recommendation is not consent. If the user declines after HEAD changes,
preserve the prior evidence but report the new HEAD as uncertified. A
same-HEAD `minikube-t2-runtime` retry finishes an authorized run without
repeating T0/T1.

## Step 3 — T1 specifics (Real PostgreSQL)

Expand Down
2 changes: 1 addition & 1 deletion .cursor/skills/minikube-t0-t1-t2/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ Companion to `SKILL.md`. Source of truth: `scripts/minikube/t2.sh`,
| `ZERO_TESTS_EXECUTED` | A lane was configured off or executed nothing (including a required-but-missing Playwright journey). | Re-run with the lane enabled, or supply the required `T2_PLAYWRIGHT_COMMAND`. |
| `REAL_PG_REQUIRED_BUT_UNAVAILABLE` | The explicit context, isolated PostgreSQL, DSN route, or exact-head database precondition is unavailable. | Re-enter `make minikube-t2` or `make minikube-t2-real-postgres` on the owned profile. Do not `docker run`, restart Docker Desktop, or probe published ports. |
| `REAL_PG_REPORT_INCOMPLETE` | Vitest JSON did not prove the exact expected physical files. | Repair test selection/reporter output; do not accept suite counters as file identity. |
| `REAL_PG_SUITE_FAILED` | Reporter, test assertions, process exit, or parent result contract failed. | Iterate with standalone T1; once green, run full T2 for final exact-head lane evidence. |
| `REAL_PG_SUITE_FAILED` | Reporter, test assertions, process exit, or parent result contract failed. | Iterate with standalone T1; once green, run full T2 for final exact-head lane evidence. If Darwin 27 reports `ECONNRESET` in `routes.externalGfsRateLimitStress.realPostgres.integration.test.ts`, check the suite's shared `http.Agent`: it must cap `maxSockets` below macOS `kern.ipc.somaxconn` (the proven value is 64). An unbounded 481-request burst is reset before Express accepts it; this is a test-socket defect, not a PostgreSQL or Minikube failure. |
| `POSTGRES_NOT_READY` | PostgreSQL precondition failed, or a PVC reset was requested without the exact expected UID. | Fix DB readiness; never guess a PVC UID. |
| `PROFILE_UNHEALTHY` | A required deployment is unready in a fail-loud check (`T2_PLAN_MODE=false`: standalone preflight or the final exact-head T2), the mandatory targeted-sync health command is missing, or a supplied user-facing health command failed or timed out. The orchestrator planner never emits this merely for an unready deployment — it selects `full-reconcile` instead. | For a targeted sync, supply the affected service's profile-owned `T2_HEALTHCHECK_COMMAND`; otherwise repair the reported health failure. Re-run on the same HEAD and profile. Do not run manual repair scripts between runs. |
| `NP08_HCC_AUTHORIZATION_FAILED` | The deployed Host-to-HCC authorization journey or cleanup failed after T0/T1 passed. | Repair the first NP08 failure, then use `minikube-t2-runtime` on the exact same profile/HEAD; never refresh/reissue in the test. |
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/ci-public.yml
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,9 @@ jobs:
- name: Test HCC watch-recovery gate coordination
run: bash scripts/tests/test-hcc-watch-recovery-gate.sh

- name: Test autoresearch gate wrappers
run: bash scripts/tests/test-autoresearch-gates.sh

- name: Test HCC watch recovery TLS proxy and configuration restoration
run: |
bash scripts/tests/test-hcc-watch-api-proxy.sh
Expand All @@ -542,6 +545,9 @@ jobs:
- name: Test HCC watch-churn readiness gate contract
run: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh

- name: Test HCC stateless cache-containment gate contract
run: bash scripts/tests/test-hcc-stateless-cache-containment.sh

- name: HCC policy lifecycle and restoration contracts
run: |
bash scripts/tests/test-hcc-watch-churn-empty-fleet.sh
Expand Down
20 changes: 20 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,26 @@ T2 with `make minikube-t2-runtime` (`T2_RUN_T0=false T2_RUN_T1=false`). That
path is valid only when the pre-gate marker already matches HEAD
(`already-synced`). Do not set those flags to skip an uncertified lane.

Before repeating an already completed full T0/T1/T2 certification, propose
the repeat with its reason and expected cost, then ask the user whether to run
it. A recommendation alone does not authorize a repeat. This choice does not
turn an earlier HEAD's evidence into certification for a newer HEAD: if the
user declines, report the newer HEAD as uncertified. A same-HEAD
`minikube-t2-runtime` retry to finish an authorized run is the supported
recovery path, not a new full recertification.

For branch-owned user-facing E2E health journeys, resolve the seeded test
credential locally before patching HCC or creates fault-injection fixtures.
Use `scripts/e2e/load-dotenv.sh` with `dotenv_load_canonical_root` and
`scripts/e2e/admin-credentials.sh` with `e2e_resolve_admin_password`; an
explicit `E2E_USER_PASSWORD` may override that journey's credential. Keep the
resolved value in process memory. Never inspect, print, log, or persist `.env`
contents, even to test whether a key exists. Missing credentials must fail
preflight before mutating the owned profile. Use the profile's persisted
ports and the exact `T2_PORT_FORWARD_COMMAND` hold when `pre-gate-sync` can
restart a service; `targeted-sync` also requires a bounded
`T2_HEALTHCHECK_COMMAND`.

The active profile mutation lock is `$T2_LOCK_ROOT/<profile>.lock`; stale-lock
reclaim coordination uses the sibling directory
`$T2_LOCK_ROOT/<profile>.reclaim`. A killed reclaimer may leave that sibling
Expand Down
21 changes: 19 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ test-unit-all: test-service-matrix ## Run unit tests across all services
test-service-matrix: ## Check local test coverage against the CI service matrix
@node scripts/dev/check-test-services.cjs $(TEST_SERVICES)

.PHONY: autoresearch-checks
autoresearch-checks: ## Run the local checks used by AutoResearch hard gates
@bash scripts/dev/autoresearch-checks.sh

.PHONY: test-codex-subscription-t0
test-codex-subscription-t0: ## Run the Codex subscription T0 aggregator (counts, no skips)
@bash scripts/tests/test-codex-subscription-t0.sh
Expand Down Expand Up @@ -784,6 +788,19 @@ minikube-t2-hcc-watch-recovery: ## Certify PR A recovery omission, runtime repai
@bash scripts/tests/test-hcc-watch-pr-a.sh
@E2E_HCC_PR_A=1 $(MAKE) minikube-t2-hcc-networkpolicy-lifecycle

.PHONY: minikube-t2-hcc-stateless-cache-containment
minikube-t2-hcc-stateless-cache-containment: ## Opt-in branch-owned T2 cache-containment journey; requires matching profile/context and refreshed user-facing forwards
@bash scripts/tests/test-hcc-stateless-cache-containment.sh
@context='$(CONTROL_API_REAL_PG_CONTEXT)'; \
[ -n "$$context" ] || { echo 'Set CONTROL_API_REAL_PG_CONTEXT to the verified branch-owned context' >&2; exit 1; }; \
[ "$(MINIKUBE_PROFILE)" = "$$context" ] || { echo 'MINIKUBE_PROFILE and CONTROL_API_REAL_PG_CONTEXT must select the same branch-owned profile/context' >&2; exit 1; }; \
printf '%s\n' "$$context" | grep -Eq '^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$$' || { echo 'The selected context must be generated and branch-scoped' >&2; exit 1; }
@E2E_HCC_WATCH_FAULT_INJECTION=1 \
T2_GATE_ID=minikube-t2 \
T2_HEALTHCHECK_COMMAND='KUBECONTEXT="$$T2_CONTEXT" MINIKUBE_PROFILE="$$T2_PROFILE" E2E_BRANCH_PROFILE_ENV="$$T2_PROFILE_ENV" E2E_PROFILE_PORTS_ENV="$$T2_PORTS_ENV" E2E_EXPECTED_PRE_GATE_GATE=minikube-t2 bash scripts/e2e/e2e-hcc-stateless-cache-containment.sh' \
T2_HEALTHCHECK_TIMEOUT_SECONDS=900 T2_HEALTHCHECK_KILL_GRACE_SECONDS=300 \
$(MAKE) minikube-t2

.PHONY: minikube-t2-runtime
minikube-t2-runtime: ## Exact-head T2 after T0 and T1 already passed on this HEAD and profile
@T2_RUN_T0=false T2_RUN_T1=false \
Expand Down Expand Up @@ -1373,12 +1390,12 @@ test-e2e-stateless-durability: ## Run stateless host durability E2E gate (Phase
.PHONY: test-e2e-stateless-suspend-wake
test-e2e-stateless-suspend-wake: ## Run stateless suspend/wake E2E gate (Phase 1 API + Phase 2 Desktop Playwright; needs port-forwards + seeded chatllm-stateless)
@echo "Running stateless suspend/wake E2E gate..."
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-suspend-wake.sh
E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-suspend-wake.sh

.PHONY: test-e2e-stateless-wake-recovery
test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R1 warm draining / R2 cold suspended / R3 drained-window p95 budgets; needs port-forwards + seeded chatllm-stateless)
@echo "Running stateless wake-recovery latency gate..."
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-wake-recovery.sh
E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-wake-recovery.sh

.PHONY: test-e2e-hcc-communicationchannel-watch-recovery
test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC watch-recovery fault-injection gate
Expand Down
4 changes: 2 additions & 2 deletions control-api/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion control-api/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
"version": "0.1.680",
"version": "0.1.681",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
Expand Down
6 changes: 3 additions & 3 deletions control-api/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1145,11 +1145,11 @@ export const config: Config = {
),
// Default derived from the wake mechanism's worst case, not picked ad hoc.
// rpc-proxy's wake-and-hold loop re-triggers POST /rpc/hosts/:hostRef/wake
// every wakeRetriggerMs=15000 for up to wakeMaxHoldMs=90000 (defaults in
// every wakeRetriggerMs=15000 for up to wakeMaxHoldMs=48000 (defaults in
// rpc-proxy/src/config.ts), so one held request costs
// 1 + floor(90000/15000) = 7 calls per rpc-proxy instance; rpc-proxy runs
// 1 + floor(48000/15000) = 4 calls per rpc-proxy instance; rpc-proxy runs
// replicas: 2 (deploy/base/rpc-proxy/rpc-proxy.yaml) with per-instance
// dedup, so the mechanism alone can emit 14 calls / 90s (~9.4/min) per
// dedup, so the mechanism alone can emit 8 calls / 48s (10.0/min) per
// host. Desktop prewarm adds a burst of up to 3 calls per device.
// Middleware order in routes/rpc-access/hosts.ts is limiter BEFORE
// handler, and the hostWakeCoalesceWindowMs coalescer runs INSIDE the
Expand Down
27 changes: 17 additions & 10 deletions control-api/src/routes/rpc-access/users.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import express, { Router } from 'express'
import type { Request } from 'express'
import type { Request, Response } from 'express'
import { config } from '../../config.js'
import { pool } from '../../db.js'
import { K8sGateway } from '../../k8s.js'
Expand Down Expand Up @@ -48,14 +48,25 @@ const HOST_ACCESS_SCOPES = [

type RpcAuthedRequest = Request & { rpcAuth?: RpcAccessClaims }

function logHostAccessDenial(
/**
* Response header that tells rpc-proxy why control-api denied Host access. The
* 403 body stays exactly `{"error":"Forbidden"}` (a pinned 21-byte contract);
* the reason travels out of band so rpc-proxy can tell removed access from
* other denials. Scope denials from the auth middleware carry no such header.
*/
export const HOST_ACCESS_DENIAL_REASON_HEADER = 'x-host-access-denial-reason'

function denyHostAccess(
req: RpcAuthedRequest,
res: Response,
reason: RpcHostAccessDenialReason | 'claims_missing'
): void {
req.log?.warn(
{ event: 'rpc_host_access_denied', reason },
'rpc host access denied by control-plane authority'
)
res.setHeader(HOST_ACCESS_DENIAL_REASON_HEADER, reason)
res.status(403).json({ error: 'Forbidden' })
}

async function bindDirectRunWithinBudget(
Expand Down Expand Up @@ -187,8 +198,7 @@ export function createRpcAccessUsersRouter(
const hostRef = String(req.params.hostRef || '').trim()
const claims = req.rpcAuth
if (!claims) {
logHostAccessDenial(req, 'claims_missing')
res.status(403).json({ error: 'Forbidden' })
denyHostAccess(req, res, 'claims_missing')
return
}
const authorization = await authorizeRpcHostAccess(
Expand All @@ -199,8 +209,7 @@ export function createRpcAccessUsersRouter(
directory
)
if (!authorization.authorized) {
logHostAccessDenial(req, authorization.reason)
res.status(403).json({ error: 'Forbidden' })
denyHostAccess(req, res, authorization.reason)
return
}
res.status(200).json(authorization.connection)
Expand All @@ -221,8 +230,7 @@ export function createRpcAccessUsersRouter(
const claims = req.rpcAuth
const binding = parseDirectRunBindingRequest(req.body)
if (!claims) {
logHostAccessDenial(req, 'claims_missing')
res.status(403).json({ error: 'Forbidden' })
denyHostAccess(req, res, 'claims_missing')
return
}
if (!binding) {
Expand All @@ -238,8 +246,7 @@ export function createRpcAccessUsersRouter(
directory
)
if (!authorization.authorized) {
logHostAccessDenial(req, authorization.reason)
res.status(403).json({ error: 'Forbidden' })
denyHostAccess(req, res, authorization.reason)
return
}

Expand Down
14 changes: 7 additions & 7 deletions control-api/test/config.hostWakeRateLimit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ import { readFileSync } from 'node:fs'

// Hard-coded mirrors (source files named above).
const WAKE_RETRIGGER_MS = 15_000
const WAKE_MAX_HOLD_MS = 90_000
const WAKE_MAX_HOLD_MS = 48_000
const RPC_PROXY_REPLICAS = 2
const PREWARM_CALLS_PER_DEVICE = 3 // 1 initial + PREWARM_REEMIT_MAX_ATTEMPTS
const PREWARM_CONCURRENT_DEVICES_BUDGET = 4

// Worst case of the wake-and-hold mechanism alone, in raw calls:
// per instance: 1 initial + floor(maxHold / retrigger) retriggers = 7
// fleet-wide: x replicas (per-instance dedup, no cross-instance dedup)
// per minute: 14 calls / 90s = ~9.33/min
// per instance: 1 initial + floor(maxHold / retrigger) retriggers = 4
// fleet-wide: x replicas (per-instance dedup, no cross-instance dedup) = 8
// per minute: 8 calls / 48s = 10.0/min
const HOLD_CALLS_PER_INSTANCE = 1 + Math.floor(WAKE_MAX_HOLD_MS / WAKE_RETRIGGER_MS)
const HOLD_CALLS_FLEET = HOLD_CALLS_PER_INSTANCE * RPC_PROXY_REPLICAS
const MECHANISM_CALLS_PER_MIN = HOLD_CALLS_FLEET * (60_000 / WAKE_MAX_HOLD_MS)
Expand Down Expand Up @@ -122,9 +122,9 @@ describe('host-wake rate limit budget (cross-service derivation guard)', () => {
const config = await loadConfigWith({})

// Sanity-pin the derivation arithmetic itself.
expect(HOLD_CALLS_PER_INSTANCE).toBe(7)
expect(HOLD_CALLS_FLEET).toBe(14)
expect(MECHANISM_CALLS_PER_MIN).toBeCloseTo(9.333, 3)
expect(HOLD_CALLS_PER_INSTANCE).toBe(4)
expect(HOLD_CALLS_FLEET).toBe(8)
expect(MECHANISM_CALLS_PER_MIN).toBeCloseTo(10.0, 3)

// Invariant 1: >=3x the single-hold mechanism volume, so the backstop
// alone can never come close to exhausting the bucket.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import request from 'supertest'
import { createApp } from '../src/app.js'
Expand Down Expand Up @@ -63,6 +63,8 @@ vi.mock('../src/db.js', () => ({
describe('Workflow approval refresh rate limit', () => {
let app: ReturnType<typeof createApp>

afterEach(() => vi.restoreAllMocks())

beforeEach(() => {
vi.clearAllMocks()
rateLimitCounts.clear()
Expand All @@ -72,6 +74,9 @@ describe('Workflow approval refresh rate limit', () => {

it('does not consume a refresh token when the request is rate limited', async () => {
const limit = config.approvalRlRefreshPerMin
// Pin the fixed-window clock before warm-up, including near a minute boundary.
const baseNow = Math.floor(Date.now() / 60_000) * 60_000 + 10_000
const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(baseNow)
// Warm up exactly `limit` calls with unique refresh tokens — all should 200.
for (let i = 0; i < limit; i++) {
const { token } = issueMcpHostRefreshJwt('ns', 'recipe')
Expand All @@ -94,8 +99,6 @@ describe('Workflow approval refresh rate limit', () => {
// Rollover the 60s window so the limiter grants a fresh slot. The
// rateLimiterService uses `Date.now()` internally — spy on it so we don't
// actually wait a minute in the test.
const baseNow = Date.now()
const nowSpy = vi.spyOn(Date, 'now')
nowSpy.mockReturnValue(baseNow + 60_001)

try {
Expand Down
Loading
Loading