From 9cb87e21dfb4732311fe9fb9e113fa9299ec0de5 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:57:20 -0500 Subject: [PATCH] fix: hub registration key rotation with grace window Co-Authored-By: Claude Opus 5.5 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .../added-19235-hub-key-rotation-grace.md | 1 + src/docs/api-reference/hub-server.md | 4 +- src/docs/audit-log.md | 2 +- src/docs/env-vars.md | 2 + src/docs/hub-deployment.md | 8 +- src/pkg/hub/selfservice_api.go | 7 +- src/pkg/hub/selfservice_api_test.go | 2 + src/pkg/hub/selfservice_key_grace.go | 131 +++++++++++ src/pkg/hub/selfservice_key_grace_test.go | 205 ++++++++++++++++++ src/pkg/hub/selfservice_registry.go | 32 +-- src/pkg/hub/selfservice_v1_api.go | 15 +- 11 files changed, 367 insertions(+), 42 deletions(-) create mode 100644 changelog.d/added-19235-hub-key-rotation-grace.md create mode 100644 src/pkg/hub/selfservice_key_grace.go create mode 100644 src/pkg/hub/selfservice_key_grace_test.go diff --git a/changelog.d/added-19235-hub-key-rotation-grace.md b/changelog.d/added-19235-hub-key-rotation-grace.md new file mode 100644 index 0000000000..993527f045 --- /dev/null +++ b/changelog.d/added-19235-hub-key-rotation-grace.md @@ -0,0 +1 @@ +- **Hub: registration key rotation with a grace window ([#19235](https://github.com/hivecommons/hive/issues/19235)).** With `HIVE_HUB_SELFSERVICE_KEY_GRACE` on, the old heartbeat key of a self-service registration keeps working for `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` (default `1h`, clamped to `168h`) after a rotation, and rotating again inside the window keeps the old key and replaces the pending one, so a rotation never cuts off a running spoke. Off by default; existing registrations need no change and nothing writes to GitHub. The `selfservice_hive_rotated` audit line gains `grace`, `replaced_pending` and `old_key_expires_at`, with no key or hash. diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index d323183f8d..3819973536 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -63,9 +63,9 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/hub/v1/registrations` | Hub auth | The caller's own registrations: `{registrations[]}`, each with its `etag` and `repo_proofs`; never a key or a hash | `pkg/hub/selfservice_v1_api.go` | | `GET` | `/api/hub/v1/registrations/{id}` | Hub auth | One of the caller's registrations: `{registration}` with a strong `ETag` header; `If-None-Match` with the current ETag answers `304`. Another user's hive, a missing hive and a hub admin who does not own it all get `404`. The ETag comes from a revision counter that heartbeats do not bump, and reveals nothing about the key | `pkg/hub/selfservice_v1_api.go` | | `PATCH` | `/api/hub/v1/registrations/{id}` | Hub auth | Owner only, by hub session (a registration key is not accepted): change `repos` and/or `dashboard_url` in place with the heartbeat key kept valid ([#18642](https://github.com/hivecommons/hive/issues/18642)). Body: `repos` is a full list (replacement) or `{add[], remove[]}`; `dashboard_url` is an `https` URL or `""` to clear it. `If-Match` is required: `428` when absent, `412` carrying the current `ETag` when stale (checked before any GitHub call and again under the store lock). Only added repos are proven (`403` naming the first refused repo, nothing applied); kept repos keep their proof time and removed repos lose theirs. `org`, the owner, the ID and the keys are not editable: a body naming `org` or any other field is `400`, as is a list that would leave no repo. Over `HIVE_HUB_SELFSERVICE_MAX_REPOS` is `413 {error, limit: "max_repos", max_repos}`. Removing a repo the registration does not list is ignored; an edit that changes nothing answers `200` with the current `ETag` and does not bump the revision. `200 {registration}` with the new `ETag`; the key never appears. `404` for a missing hive, another user's hive and a hub admin who does not own it; `404` while `HIVE_HUB_SELFSERVICE_REGISTRATION` is off | `pkg/hub/selfservice_v1_patch.go` | -| `POST` | `/api/hub/v1/registrations/{id}/keys` | Hub auth | Owner only: rotate the heartbeat key; the previous key keeps authenticating for the 10 minute overlap. `201 {id, key, hub_url, overlap, snippets}` (the legacy rotate answers `200`) with the new `ETag`; the raw key appears once. Optional `If-Match` (`412` when stale). A body asking for a grace window is refused with `400`: caller-chosen grace is a later rollout step | `pkg/hub/selfservice_v1_api.go` | +| `POST` | `/api/hub/v1/registrations/{id}/keys` | Hub auth | Owner only: rotate the heartbeat key; the previous key keeps authenticating for the 10 minute overlap, or for `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` while `HIVE_HUB_SELFSERVICE_KEY_GRACE` is on, in which case a rotation inside an open window keeps the old key and replaces the pending one ([#19235](https://github.com/hivecommons/hive/issues/19235)). `201 {id, key, hub_url, overlap, snippets}` (the legacy rotate answers `200`) with the new `ETag`; the raw key appears once. Optional `If-Match` (`412` when stale). A body asking for a grace window is refused with `400`: caller-chosen grace is a later rollout step | `pkg/hub/selfservice_v1_api.go` | | `DELETE` | `/api/hub/v1/registrations/{id}` | Hub auth | Owner or hub admin: revoke the key and remove the registry entry. `200 {removed, id}`. Optional `If-Match` (`412` when stale, carrying the current `ETag`) | `pkg/hub/selfservice_v1_api.go` | -| `POST` | `/api/hub/selfservice/hives/{id}/rotate` | Hub auth | Owner only: mint a new heartbeat key for `{id}`; the previous key keeps authenticating for `overlap` (10 minutes). `200 {id, key, hub_url, overlap, snippets}`, raw key once. Another user's hive, a missing hive and a hub admin who does not own it all get `404` | `pkg/hub/server.go:1794` | +| `POST` | `/api/hub/selfservice/hives/{id}/rotate` | Hub auth | Owner only: mint a new heartbeat key for `{id}`; the previous key keeps authenticating for `overlap` (10 minutes, or the grace window while `HIVE_HUB_SELFSERVICE_KEY_GRACE` is on). `200 {id, key, hub_url, overlap, snippets}`, raw key once. Another user's hive, a missing hive and a hub admin who does not own it all get `404` | `pkg/hub/server.go:1794` | | `DELETE` | `/api/hub/selfservice/hives/{id}` | Hub auth | Owner or hub admin: revoke the key and remove the registry entry. `200 {removed, id}`; another user's hive is `404` | `pkg/hub/server.go:1795` | | `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1799` | | `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1800` | diff --git a/src/docs/audit-log.md b/src/docs/audit-log.md index 8f52b17f41..5c24dfe658 100644 --- a/src/docs/audit-log.md +++ b/src/docs/audit-log.md @@ -205,7 +205,7 @@ a key hash, the master secret or the user's GitHub token. | --- | --- | --- | | `selfservice_hive_registered` | A registration passed every check and the key was minted | `user`, `hive`, `repos` (count), `org` | | `selfservice_hive_updated` | The owner changed `repos` and/or `dashboard_url` with `PATCH /api/hub/v1/registrations/{id}`; the heartbeat key is untouched. A request that changes nothing writes no line | `user`, `hive`, `repos_added` (count), `repos_removed` (count), `dashboard_changed`, `revision` | -| `selfservice_hive_rotated` | The owner rotated the heartbeat key | `user`, `hive`, `overlap` (how long the previous key keeps authenticating) | +| `selfservice_hive_rotated` | The owner rotated the heartbeat key | `user`, `hive`, `overlap` (how long the previous key keeps authenticating), `grace` (`HIVE_HUB_SELFSERVICE_KEY_GRACE` on), `replaced_pending` (a rotation inside an open grace window replaced the pending key and kept the old one), `old_key_expires_at` (RFC 3339). Never a key or a hash | | `selfservice_hive_revoked` | The owner or a hub admin revoked the hive; the registry entry is removed in the same request and `registry entry removed` follows | `user`, `hive`, `owner`, `by_admin` | | `selfservice_hive_expired` | The expiry sweep (`pkg/hub/selfservice_expiry.go`, every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL`) removed a hive that went longer than `HIVE_HUB_SELFSERVICE_EXPIRY` without an accepted heartbeat; the key is revoked and `registry entry removed` (with `by=selfservice-expiry-sweep`) follows | `id`, `owner`, `last_heartbeat` (RFC 3339, or `never` when no heartbeat was ever accepted), `expiry` | | `selfservice_hive_refused` | A registration was turned away and nothing was stored | `user`, `hive`, `reason` (`invalid_name`, `invalid_login`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `invalid_update` (a `PATCH` body the hub refused), `per_user_cap` with `max_per_user`, `no_github_token`, `no_github_login`, `not_repo_admin` with `repo`, `ownership_check_error` with `repo`, `proof_token_refused` (a failed `github-user-token` proof) with `proof_method`, `proof_reason` and `repo` when one repo failed) | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 4f31f21203..943f5d3280 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -126,6 +126,8 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | No | `5` | Maximum self-service hives one user may have registered at once; the next registration answers `429` until one is removed. | | `HIVE_HUB_SELFSERVICE_MAX_REPOS` | No | `20` | Maximum repos one self-service registration may list ([#18642](https://github.com/hivecommons/hive/issues/18642)). Each listed repo costs one read-only GitHub ownership check, so values above `500` are clamped to `500`; a non-numeric or non-positive value falls back to `20`. Advertised by `GET /api/hub/v1` as `limits.max_repos`. Over the cap, `POST /api/hub/selfservice/hives` answers `400`; `POST /api/hub/v1/registrations` answers `413` with `limit: max_repos`. Read per request. Only meaningful while `HIVE_HUB_SELFSERVICE_REGISTRATION` is on. | | `HIVE_HUB_PROOF_GITHUB_USER_TOKEN` | No | off | Offers the `github-user-token` ownership proof on the v1 registration create and `PATCH` routes ([#19238](https://github.com/hivecommons/hive/issues/19238)): the caller sends `X-Hive-Proof-Method: github-user-token` and `X-Hive-Proof-Token`, and the hub checks `permissions.admin` with that token for the request only (never stored or logged). Advertised by `GET /api/hub/v1` in `proof_methods` and `proof_method_details`. Accepts `1/true/yes/on`. Read per request. Only meaningful while `HIVE_HUB_SELFSERVICE_REGISTRATION` is on. | +| `HIVE_HUB_SELFSERVICE_KEY_GRACE` | No | off | Turns on key rotation with a grace window for self-service registrations ([#19235](https://github.com/hivecommons/hive/issues/19235)). While on, the old key keeps authenticating for `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` after a rotation, and a rotation while a window is open keeps the old key and replaces the pending one. While off, rotation keeps the fixed 10 minute overlap. Accepts `1/true/yes/on`. Read per request. | +| `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` | No | `1h` | How long the old key keeps working after a rotation while `HIVE_HUB_SELFSERVICE_KEY_GRACE` is on. Go duration syntax; an invalid or non-positive value falls back to `1h`, and values above `168h` are clamped to `168h`. Advertised by `GET /api/hub/v1` as `limits.key_rotation_overlap`. Read per request. | | `HIVE_HUB_SELFSERVICE_EXPIRY` | No | `24h` | How long a self-service hive may go without an accepted heartbeat before the expiry sweep revokes its key and removes its registry entry (counted from the last accepted heartbeat, or from registration while there has been none). Reported by `GET /api/hub/selfservice/status` as `expiry`. Go duration syntax. | | `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` | No | `10m` | How often the self-service expiry sweep (`src/pkg/hub/selfservice_expiry.go`) runs. Each removal writes `audit: selfservice_hive_expired` (see [audit-log.md](audit-log.md)). Read at startup; restart after changing. Go duration syntax. | | `HIVE_HUB_HEARTBEAT_PROTOCOL` | No | off | Heartbeat feature negotiation ([#19237](https://github.com/hivecommons/hive/issues/19237)). When on (`1/true/yes/on`), a heartbeat that carries `protocol: {version, features}` is answered with `protocol: {version, accepted, ignored}` (plus `unsupported_version` for a version below `1`). A heartbeat without `protocol` is answered exactly as before. Read per request. See [hub-deployment.md](hub-deployment.md#heartbeat-feature-negotiation). | diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index f136b69098..e8e64b5f78 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -183,10 +183,12 @@ The fence above is always on. The registration flow that fills the `byo-` namesp | `HIVE_HUB_SELFSERVICE_MAX_PER_USER` | `5` | How many `byo-` hives one GitHub login may hold at once. The next registration answers `429` until the user removes one. | | `HIVE_HUB_SELFSERVICE_MAX_REPOS` | `20` | How many repos one registration may list ([#18642](https://github.com/hivecommons/hive/issues/18642)). Each repo is one read-only GitHub ownership check, so values above `500` are clamped to `500`. `POST /api/hub/selfservice/hives` answers `400` over the cap; `POST /api/hub/v1/registrations` answers `413` with `limit: max_repos`. Advertised to clients by `GET /api/hub/v1` as `limits.max_repos`. Read per request. | | `HIVE_HUB_PROOF_GITHUB_USER_TOKEN` | off | Offers the `github-user-token` ownership proof on `POST /api/hub/v1/registrations` and `PATCH /api/hub/v1/registrations/{id}` ([#19238](https://github.com/hivecommons/hive/issues/19238)). See [Proving ownership with your own token](#proving-ownership-with-your-own-token). Accepts `1/true/yes/on`. Read per request. Only meaningful while `HIVE_HUB_SELFSERVICE_REGISTRATION` is on. | +| `HIVE_HUB_SELFSERVICE_KEY_GRACE` | off | Key rotation with a grace window ([#19235](https://github.com/hivecommons/hive/issues/19235)); see [Rotation](#rotation). Accepts `1/true/yes/on`. Read per request. | +| `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` | `1h` | How long the old key keeps working after a rotation while `HIVE_HUB_SELFSERVICE_KEY_GRACE` is on. Clamped to `168h`. Read per request. | | `HIVE_HUB_SELFSERVICE_EXPIRY` | `24h` | How long a registered hive may go without an accepted heartbeat before the sweep removes it. Counted from the last accepted heartbeat, or from registration while there has been none. | | `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` | `10m` | How often the expiry sweep runs. Read at startup; restart after changing. | -All four are listed with the other hub variables in [env-vars.md](env-vars.md). +All of them are listed with the other hub variables in [env-vars.md](env-vars.md). #### What a registration does @@ -245,6 +247,8 @@ A spoke may send `protocol: {"version": 1, "features": [...]}` in its heartbeat The owner can rotate from the **Self-hosted hives** list or with `POST /api/hub/selfservice/hives/{id}/rotate`. The hub mints a new key and keeps the previous hash live for a 10 minute overlap so a running spoke can pick up the new value without a gap. After the overlap only the new key is accepted. Only the owner can rotate; a hub admin who does not own the hive gets `404`, the same answer as for a missing hive. +With `HIVE_HUB_SELFSERVICE_KEY_GRACE` on ([#19235](https://github.com/hivecommons/hive/issues/19235)), the overlap is the configurable grace window `HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW` (default `1h`) instead of 10 minutes, on both the v6 route and `POST /api/hub/v1/registrations/{id}/keys`. Rotating again while a window is open keeps the old key, the one the spoke is still running, and replaces the pending key: the key from the earlier rotation stops working at once, and the window restarts. So rotating twice by mistake never cuts off the running spoke. The `overlap` in the response and `limits.key_rotation_overlap` in discovery report the window in effect. Existing registrations need no change. + #### Expiry sweep A background sweep runs every `HIVE_HUB_SELFSERVICE_SWEEP_INTERVAL` and removes any `byo-` hive whose last accepted heartbeat is older than `HIVE_HUB_SELFSERVICE_EXPIRY`, or that never heartbeated within that window after registration. The key is revoked and the registry entry is removed in the same pass. The owner registers again when the hive is back; there is no grace list. @@ -262,7 +266,7 @@ Every decision writes one structured log line, documented in [audit-log.md](audi | `selfservice_hive_registered` | A registration passed every check and a key was minted. | | `selfservice_hive_refused` | A registration was turned away; `reason` is one of `invalid_name`, `reserved_prefix`, `duplicate_id`, `invalid_repos`, `invalid_dashboard_url`, `invalid_update` (a `PATCH` body that was refused), `per_user_cap`, `no_github_token`, `no_github_login`, `not_repo_admin`, `ownership_check_error`. | | `selfservice_hive_updated` | The owner edited the repos or dashboard URL with `PATCH`; the line counts repos added and removed and carries no key. | -| `selfservice_hive_rotated` | The owner rotated the key; `overlap` is how long the old key keeps working. | +| `selfservice_hive_rotated` | The owner rotated the key; `overlap` is how long the old key keeps working, `grace` whether the grace window is on, `replaced_pending` whether a pending key inside an open window was replaced, `old_key_expires_at` when the old key stops working. No key or hash. | | `selfservice_hive_revoked` | The owner or an admin removed the hive; `by_admin` tells which. | | `selfservice_hive_expired` | The sweep removed a hive; `last_heartbeat` is RFC 3339 or `never`. | diff --git a/src/pkg/hub/selfservice_api.go b/src/pkg/hub/selfservice_api.go index 61e1fea8a6..d19c7a7de4 100644 --- a/src/pkg/hub/selfservice_api.go +++ b/src/pkg/hub/selfservice_api.go @@ -688,7 +688,8 @@ func (s *HubServer) handleSelfServiceRotate(w http.ResponseWriter, r *http.Reque if !ok { return } - rawKey, err := s.selfService.rotate(h.ID) + policy := currentSelfServiceKeyRotationPolicy() + rawKey, rotation, err := s.selfService.rotateWithPolicy(h.ID, "", policy) if err != nil { if errors.Is(err, errSelfServiceNotFound) { writeJSONError(w, http.StatusNotFound, "self-hosted hive not found") @@ -698,14 +699,14 @@ func (s *HubServer) handleSelfServiceRotate(w http.ResponseWriter, r *http.Reque writeJSONError(w, http.StatusInternalServerError, "could not persist the new key") return } - s.logger.Info("audit: selfservice_hive_rotated", "user", login, "hive", h.ID, "overlap", selfServiceKeyRotationOverlap.String()) + s.auditSelfServiceRotated(login, h.ID, policy, rotation) hubURL := hubPublicURL() w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "id": h.ID, "key": rawKey, "hub_url": hubURL, - "overlap": selfServiceKeyRotationOverlap.String(), + "overlap": policy.window.String(), "snippets": selfServiceSnippets(hubURL, h.ID, rawKey), }) } diff --git a/src/pkg/hub/selfservice_api_test.go b/src/pkg/hub/selfservice_api_test.go index da311bed28..a51d708473 100644 --- a/src/pkg/hub/selfservice_api_test.go +++ b/src/pkg/hub/selfservice_api_test.go @@ -62,6 +62,8 @@ func newSelfServiceTestEnv(t *testing.T) *selfServiceTestEnv { t.Helper() t.Setenv(selfServiceRegistrationEnv, "true") t.Setenv(selfServiceMaxPerUserEnv, "") + t.Setenv(selfServiceKeyGraceEnv, "") + t.Setenv(selfServiceKeyGraceWindowEnv, "") t.Setenv(selfServiceTestAdminEnv, selfServiceTestAdmin) withTempSaaSDirs(t) diff --git a/src/pkg/hub/selfservice_key_grace.go b/src/pkg/hub/selfservice_key_grace.go new file mode 100644 index 0000000000..3efa0eef81 --- /dev/null +++ b/src/pkg/hub/selfservice_key_grace.go @@ -0,0 +1,131 @@ +package hub + +// Hub registration protocol v1, rollout step 2: key rotation with a grace +// window (hivecommons/hive#19235, #19056). +// +// Rotation has always kept the previous key live for the fixed +// selfServiceKeyRotationOverlap. That is too short for an owner who redeploys +// a spoke by hand, and a second rotation inside the overlap made the previous +// key the one minted a moment earlier, so the key the spoke was still running +// stopped working at once: a rotation could cause an outage. +// +// With HIVE_HUB_SELFSERVICE_KEY_GRACE on: +// - the old key keeps authenticating for HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW +// (default selfServiceKeyGraceWindowDefault) after a rotation; +// - a rotation while a grace window is open keeps the old key (the one the +// spoke still runs) and replaces the pending new key, whose hash is +// dropped, and the window restarts from this rotation. +// +// Off (the default), rotation behaves exactly as before. Nothing here changes +// the stored record shape, so existing registrations keep working with no data +// change, and nothing here logs or returns a hash. + +import ( + "os" + "strings" + "time" +) + +const ( + // selfServiceKeyGraceEnv turns the configurable grace window on. Off by + // default: a hub keeps the fixed overlap until its operator enables it. + selfServiceKeyGraceEnv = "HIVE_HUB_SELFSERVICE_KEY_GRACE" + // selfServiceKeyGraceWindowEnv is how long the old key keeps working after + // a rotation while the grace flag is on. + selfServiceKeyGraceWindowEnv = "HIVE_HUB_SELFSERVICE_KEY_GRACE_WINDOW" + // selfServiceKeyGraceWindowDefault is the grace window when the env is + // unset or invalid. + selfServiceKeyGraceWindowDefault = time.Hour + // selfServiceKeyGraceWindowCeiling clamps a configured window so a typo + // cannot keep a rotated-away key alive for months. + selfServiceKeyGraceWindowCeiling = 7 * 24 * time.Hour +) + +// selfServiceKeyRotationPolicy is how one rotation treats the previous key. +type selfServiceKeyRotationPolicy struct { + // window is how long the old key keeps authenticating after the rotation. + window time.Duration + // replacePending keeps a still-live old key across a second rotation and + // replaces the pending key instead of making it the old one. + replacePending bool +} + +// selfServiceKeyGraceEnabled reads the flag per request, like the other +// self-service toggles. +func selfServiceKeyGraceEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(selfServiceKeyGraceEnv))) { + case "1", "true", "yes", "on": + return true + } + return false +} + +// currentSelfServiceKeyRotationPolicy is the policy rotation uses now: the +// legacy fixed overlap while the flag is off, the configured grace window +// otherwise. +func currentSelfServiceKeyRotationPolicy() selfServiceKeyRotationPolicy { + if !selfServiceKeyGraceEnabled() { + return selfServiceKeyRotationPolicy{window: selfServiceKeyRotationOverlap} + } + window := min(durationEnv(selfServiceKeyGraceWindowEnv, selfServiceKeyGraceWindowDefault), selfServiceKeyGraceWindowCeiling) + return selfServiceKeyRotationPolicy{window: window, replacePending: true} +} + +// selfServiceRotation describes a completed rotation for the audit line and +// the response. It carries no key material. +type selfServiceRotation struct { + // replacedPending is true when the rotation replaced a pending key inside + // an open grace window rather than retiring the current one. + replacedPending bool + // oldKeyExpiresAt is when the old key stops authenticating. + oldKeyExpiresAt time.Time +} + +// rotateWithPolicy mints a new key for id under policy and returns it ONCE. +// ifMatch follows update's rules; a mismatch returns errSelfServiceStale. +func (st *selfServiceStore) rotateWithPolicy(id, ifMatch string, policy selfServiceKeyRotationPolicy) (string, selfServiceRotation, error) { + if st == nil { + return "", selfServiceRotation{}, errSelfServiceNilStore + } + raw, hash, err := mintSelfServiceKey() + if err != nil { + return "", selfServiceRotation{}, err + } + st.mu.Lock() + defer st.mu.Unlock() + h, ok := st.hives[id] + if !ok { + return "", selfServiceRotation{}, errSelfServiceNotFound + } + if !selfServiceETagMatches(*h, ifMatch) { + return "", selfServiceRotation{}, errSelfServiceStale + } + now := st.clock().UTC() + before := *h + var info selfServiceRotation + if policy.replacePending && h.PrevKeyHash != "" && now.Before(h.PrevKeyExpiresAt) { + // The old key is still the one the spoke may be running: keep it and + // drop the pending key, which was never deployed long enough to matter. + info.replacedPending = true + } else { + h.PrevKeyHash = h.KeyHash + } + h.PrevKeyExpiresAt = now.Add(policy.window) + h.KeyHash = hash + h.Revision++ + if err := st.saveLocked(); err != nil { + *h = before + return "", selfServiceRotation{}, err + } + info.oldKeyExpiresAt = h.PrevKeyExpiresAt + return raw, info, nil +} + +// auditSelfServiceRotated writes the selfservice_hive_rotated line. It names +// the window and whether a pending key was replaced, never a key or a hash. +func (s *HubServer) auditSelfServiceRotated(login, hiveID string, policy selfServiceKeyRotationPolicy, rotation selfServiceRotation) { + s.logger.Info("audit: selfservice_hive_rotated", "user", login, "hive", hiveID, + "overlap", policy.window.String(), "grace", policy.replacePending, + "replaced_pending", rotation.replacedPending, + "old_key_expires_at", rotation.oldKeyExpiresAt.Format(time.RFC3339)) +} diff --git a/src/pkg/hub/selfservice_key_grace_test.go b/src/pkg/hub/selfservice_key_grace_test.go new file mode 100644 index 0000000000..a6548bb71f --- /dev/null +++ b/src/pkg/hub/selfservice_key_grace_test.go @@ -0,0 +1,205 @@ +package hub + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + "time" +) + +// graceTestPolicy is the grace policy with a window long enough to tell it +// apart from the legacy overlap. +var graceTestPolicy = selfServiceKeyRotationPolicy{window: 2 * time.Hour, replacePending: true} + +func TestSelfServiceKeyGraceBothKeysInsideWindowOldRefusedAfter(t *testing.T) { + st, now := newSelfServiceTestStore(t) + old, err := st.register(SelfServiceHive{ID: "byo-alice--lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + fresh, info, err := st.rotateWithPolicy("byo-alice--lab", "", graceTestPolicy) + if err != nil { + t.Fatalf("rotate: %v", err) + } + if info.replacedPending { + t.Fatal("first rotation reported a replaced pending key") + } + if want := now.Add(graceTestPolicy.window); !info.oldKeyExpiresAt.Equal(want) { + t.Fatalf("old key expires at %v, want %v", info.oldKeyExpiresAt, want) + } + if !st.verify("byo-alice--lab", old) || !st.verify("byo-alice--lab", fresh) { + t.Fatal("inside the grace window both keys must verify") + } + // Past the legacy overlap but inside the grace window: still both. + *now = now.Add(selfServiceKeyRotationOverlap + time.Minute) + if !st.verify("byo-alice--lab", old) { + t.Fatal("old key refused inside the grace window") + } + *now = now.Add(graceTestPolicy.window) + if st.verify("byo-alice--lab", old) { + t.Fatal("old key still verifies after the grace window") + } + if !st.verify("byo-alice--lab", fresh) { + t.Fatal("new key stopped verifying after the grace window") + } +} + +func TestSelfServiceKeyGraceSecondRotationReplacesPending(t *testing.T) { + st, now := newSelfServiceTestStore(t) + old, err := st.register(SelfServiceHive{ID: "byo-alice--lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + pending, _, err := st.rotateWithPolicy("byo-alice--lab", "", graceTestPolicy) + if err != nil { + t.Fatalf("first rotate: %v", err) + } + *now = now.Add(30 * time.Minute) + second, info, err := st.rotateWithPolicy("byo-alice--lab", "", graceTestPolicy) + if err != nil { + t.Fatalf("second rotate: %v", err) + } + if !info.replacedPending { + t.Fatal("second rotation inside the window did not report replacing the pending key") + } + if !st.verify("byo-alice--lab", old) { + t.Fatal("the original key (still deployed) was dropped by the second rotation") + } + if st.verify("byo-alice--lab", pending) { + t.Fatal("the replaced pending key still verifies") + } + if !st.verify("byo-alice--lab", second) { + t.Fatal("the newest key does not verify") + } + // The window restarts from the second rotation. + *now = now.Add(graceTestPolicy.window - time.Second) + if !st.verify("byo-alice--lab", old) { + t.Fatal("old key refused before the restarted window ended") + } + *now = now.Add(2 * time.Second) + if st.verify("byo-alice--lab", old) { + t.Fatal("old key still verifies after the restarted window") + } + // A rotation after the window closed retires the current key as usual. + third, info, err := st.rotateWithPolicy("byo-alice--lab", "", graceTestPolicy) + if err != nil { + t.Fatalf("third rotate: %v", err) + } + if info.replacedPending { + t.Fatal("rotation after the window reported a replaced pending key") + } + if !st.verify("byo-alice--lab", second) || !st.verify("byo-alice--lab", third) { + t.Fatal("after a fresh rotation the previous and new key must both verify") + } +} + +func TestSelfServiceKeyGraceOffKeepsLegacyRotation(t *testing.T) { + t.Setenv(selfServiceKeyGraceEnv, "") + t.Setenv(selfServiceKeyGraceWindowEnv, "48h") + policy := currentSelfServiceKeyRotationPolicy() + if policy.window != selfServiceKeyRotationOverlap || policy.replacePending { + t.Fatalf("flag off policy = %+v, want the legacy overlap", policy) + } + st, _ := newSelfServiceTestStore(t) + old, err := st.register(SelfServiceHive{ID: "byo-alice--lab", Owner: "alice"}) + if err != nil { + t.Fatalf("register: %v", err) + } + first, _, err := st.rotateWithPolicy("byo-alice--lab", "", policy) + if err != nil { + t.Fatal(err) + } + if _, info, err := st.rotateWithPolicy("byo-alice--lab", "", policy); err != nil || info.replacedPending { + t.Fatalf("legacy second rotate: info=%+v err=%v", info, err) + } + if st.verify("byo-alice--lab", old) { + t.Fatal("legacy second rotation must retire the original key, as before") + } + if !st.verify("byo-alice--lab", first) { + t.Fatal("legacy second rotation must keep the first rotated key in its overlap") + } +} + +func TestSelfServiceKeyGracePolicyFromEnv(t *testing.T) { + cases := []struct { + flag, window string + want time.Duration + }{ + {"true", "", selfServiceKeyGraceWindowDefault}, + {"on", "90m", 90 * time.Minute}, + {"1", "not-a-duration", selfServiceKeyGraceWindowDefault}, + {"yes", "-5m", selfServiceKeyGraceWindowDefault}, + {"true", "9000h", selfServiceKeyGraceWindowCeiling}, + } + for _, c := range cases { + t.Setenv(selfServiceKeyGraceEnv, c.flag) + t.Setenv(selfServiceKeyGraceWindowEnv, c.window) + got := currentSelfServiceKeyRotationPolicy() + if got.window != c.want || !got.replacePending { + t.Errorf("flag=%q window=%q: policy %+v, want window %v with replacePending", c.flag, c.window, got, c.want) + } + } +} + +func TestHubV1RotateKeyGraceWindow(t *testing.T) { + env := newHubV1TestEnv(t) + t.Setenv(selfServiceKeyGraceEnv, "true") + t.Setenv(selfServiceKeyGraceWindowEnv, "3h") + id, old := env.registerHappy(t) + + rotate := func() (string, string) { + t.Helper() + rec := env.v1(t, selfServiceTestUser, http.MethodPost, hubV1BasePath+"/registrations/"+id+"/keys", "", nil, env.s.handleHubV1RotateKey) + if rec.Code != http.StatusCreated { + t.Fatalf("v1 rotate: %d %s", rec.Code, rec.Body.String()) + } + var resp struct { + Key string `json:"key"` + Overlap string `json:"overlap"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + return resp.Key, resp.Overlap + } + pending, overlap := rotate() + if overlap != (3 * time.Hour).String() { + t.Fatalf("overlap = %q, want the configured window", overlap) + } + *env.now = env.now.Add(selfServiceKeyRotationOverlap + time.Minute) + if _, ok := env.s.verifyHeartbeatBearerGeneration(old, id); !ok { + t.Fatal("old key refused inside the grace window") + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(pending, id); !ok { + t.Fatal("new key refused inside the grace window") + } + newest, _ := rotate() + if _, ok := env.s.verifyHeartbeatBearerGeneration(old, id); !ok { + t.Fatal("second rotation dropped the original key") + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(pending, id); ok { + t.Fatal("replaced pending key still authenticates") + } + *env.now = env.now.Add(3*time.Hour + time.Second) + if _, ok := env.s.verifyHeartbeatBearerGeneration(old, id); ok { + t.Fatal("old key still authenticates after the grace window") + } + if _, ok := env.s.verifyHeartbeatBearerGeneration(newest, id); !ok { + t.Fatal("newest key stopped authenticating") + } + + logs := env.logs.String() + if !strings.Contains(logs, "audit: selfservice_hive_rotated") || !strings.Contains(logs, "replaced_pending=true") || !strings.Contains(logs, "grace=true") { + t.Fatalf("rotation audit line missing grace fields: %s", logs) + } + stored, _ := env.s.selfService.get(id) + for _, secret := range []string{old, pending, newest, stored.KeyHash, stored.PrevKeyHash, hashSelfServiceKey(pending)} { + if secret == "" { + t.Fatal("empty secret under test") + } + if strings.Contains(logs, secret) { + t.Fatal("a key or a key hash reached the logs") + } + } +} diff --git a/src/pkg/hub/selfservice_registry.go b/src/pkg/hub/selfservice_registry.go index ffa718c3e1..7c463ecd62 100644 --- a/src/pkg/hub/selfservice_registry.go +++ b/src/pkg/hub/selfservice_registry.go @@ -335,7 +335,9 @@ func (st *selfServiceStore) registerCapped(h SelfServiceHive, maxPerOwner int) ( } // rotate mints a new key for id and returns it ONCE. The previous key keeps -// authenticating until selfServiceKeyRotationOverlap has passed. +// authenticating until selfServiceKeyRotationOverlap has passed. The HTTP +// routes use rotateWithPolicy (selfservice_key_grace.go) so the grace flag +// applies; rotate and rotateIfMatch keep the fixed overlap. func (st *selfServiceStore) rotate(id string) (string, error) { return st.rotateIfMatch(id, "") } @@ -343,32 +345,8 @@ func (st *selfServiceStore) rotate(id string) (string, error) { // rotateIfMatch is rotate guarded by an ETag precondition (see update for // the ifMatch rules); a mismatch returns errSelfServiceStale. func (st *selfServiceStore) rotateIfMatch(id, ifMatch string) (string, error) { - if st == nil { - return "", errSelfServiceNilStore - } - raw, hash, err := mintSelfServiceKey() - if err != nil { - return "", err - } - st.mu.Lock() - defer st.mu.Unlock() - h, ok := st.hives[id] - if !ok { - return "", errSelfServiceNotFound - } - if !selfServiceETagMatches(*h, ifMatch) { - return "", errSelfServiceStale - } - before := *h - h.PrevKeyHash = h.KeyHash - h.PrevKeyExpiresAt = st.clock().UTC().Add(selfServiceKeyRotationOverlap) - h.KeyHash = hash - h.Revision++ - if err := st.saveLocked(); err != nil { - *h = before - return "", err - } - return raw, nil + raw, _, err := st.rotateWithPolicy(id, ifMatch, selfServiceKeyRotationPolicy{window: selfServiceKeyRotationOverlap}) + return raw, err } // update applies mutate to the record for id when ifMatch is empty, "*", or diff --git a/src/pkg/hub/selfservice_v1_api.go b/src/pkg/hub/selfservice_v1_api.go index ba5b326eb9..578df88944 100644 --- a/src/pkg/hub/selfservice_v1_api.go +++ b/src/pkg/hub/selfservice_v1_api.go @@ -148,7 +148,7 @@ func (s *HubServer) handleHubV1Discovery(w http.ResponseWriter, r *http.Request) "max_repos": selfServiceMaxRepos(), "max_per_user": selfServiceMaxPerUser(), "expiry": selfServiceExpiry().String(), - "key_rotation_overlap": selfServiceKeyRotationOverlap.String(), + "key_rotation_overlap": currentSelfServiceKeyRotationPolicy().window.String(), }, "proof_methods": hubV1ProofMethods(), "proof_method_details": hubV1ProofMethodDetails(), @@ -222,8 +222,8 @@ func decodeStrict(data []byte, v any) error { } // handleHubV1RotateKey is POST /api/hub/v1/registrations/{id}/keys: the v6 -// rotation (the previous key keeps working for selfServiceKeyRotationOverlap) -// with an optional If-Match. A caller-chosen grace window is rollout step 3, +// rotation (the previous key keeps working for the current rotation policy's +// window, selfservice_key_grace.go) with an optional If-Match. A caller-chosen grace window is rollout step 3, // so a body asking for one is refused rather than silently ignored. func (s *HubServer) handleHubV1RotateKey(w http.ResponseWriter, r *http.Request) { if !s.requireSelfServiceEnabled(w) { @@ -239,14 +239,15 @@ func (s *HubServer) handleHubV1RotateKey(w http.ResponseWriter, r *http.Request) writeJSONError(w, http.StatusBadRequest, "could not read the request body") return } + policy := currentSelfServiceKeyRotationPolicy() if len(bytes.TrimSpace(data)) > 0 { var body struct{} if err := decodeStrict(data, &body); err != nil { - writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("this hub takes no rotation options; the previous key stays valid for %s", selfServiceKeyRotationOverlap)) + writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("this hub takes no rotation options; the previous key stays valid for %s", policy.window)) return } } - rawKey, err := s.selfService.rotateIfMatch(h.ID, r.Header.Get("If-Match")) + rawKey, rotation, err := s.selfService.rotateWithPolicy(h.ID, r.Header.Get("If-Match"), policy) if err != nil { switch { case errors.Is(err, errSelfServiceNotFound): @@ -262,14 +263,14 @@ func (s *HubServer) handleHubV1RotateKey(w http.ResponseWriter, r *http.Request) } return } - s.logger.Info("audit: selfservice_hive_rotated", "user", login, "hive", h.ID, "overlap", selfServiceKeyRotationOverlap.String()) + s.auditSelfServiceRotated(login, h.ID, policy, rotation) stored, _ := s.selfService.get(h.ID) hubURL := hubPublicURL() writeHubV1JSON(w, http.StatusCreated, selfServiceETag(stored), map[string]any{ "id": h.ID, "key": rawKey, "hub_url": hubURL, - "overlap": selfServiceKeyRotationOverlap.String(), + "overlap": policy.window.String(), "snippets": selfServiceSnippets(hubURL, h.ID, rawKey), }) }