diff --git a/changelog.d/added-inbox-recording-backend.md b/changelog.d/added-inbox-recording-backend.md new file mode 100644 index 0000000000..a9134df769 --- /dev/null +++ b/changelog.d/added-inbox-recording-backend.md @@ -0,0 +1 @@ +- Inbox recordings, backend: a hive can now store recordings made in its own dashboard (`POST /api/inbox/recordings`, ordered 512 KiB chunks with gap detection and idempotent re-send, finalize, abort, same-origin Range playback). Owner and read-write only; every recording stores the consent notice version and the user; the raw audio is deleted after `inbox.recording.retention_days` (default 5) while the transcript and the personal-layer Knowledge fact stay; per-recording (`max_seconds` 3h, `max_bytes` 200 MiB) and hive-wide (`quota_bytes` 2 GiB) caps; idle recordings finish after 2h. Off by default (`inbox.recording.enabled`) until the dashboard recorder ships. See `src/docs/inbox-recording.md`. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 81127c8515..9866891443 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -39118,6 +39118,680 @@ } } }, + "/api/inbox/recordings": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Start a dashboard recording", + "description": "Owner or read-write (read sessions get 403). Starts a recording made in the dashboard (● Record) for the requesting user. `consent_version` must name the consent notice the dashboard showed before this recording; it is stored with the user in the recording's metadata. One recording per user may be in progress. The browser then uploads MediaRecorder chunks in order and finalizes. Audited as `inbox_recording_start`.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "mime", + "consent_version" + ], + "properties": { + "title": { + "type": "string" + }, + "mime": { + "type": "string", + "description": "MediaRecorder type: audio/webm or audio/mp4, with codecs parameters" + }, + "consent_version": { + "type": "string" + }, + "includes_tab_audio": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "201": { + "description": "Started", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "max_chunk_bytes": { + "type": "integer" + }, + "max_seconds": { + "type": "integer" + }, + "retention_days": { + "type": "integer" + }, + "transcribes": { + "type": "boolean", + "description": "The hive transcribes the recording when it ends" + } + } + } + } + } + }, + "400": { + "description": "Consent missing or stale", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not owner or read-write", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "A recording of this user is already in progress", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "expected_seq": { + "type": "integer", + "description": "Next seq the hive accepts (out-of-order chunk)" + }, + "active_id": { + "type": "string", + "description": "The user's recording already in progress" + } + } + } + } + } + }, + "412": { + "description": "inbox.recording.enabled is off", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "415": { + "description": "Not audio/webm or audio/mp4", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "507": { + "description": "inbox.recording.quota_bytes used up", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/recordings/{id}/chunks": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Upload one recording chunk", + "description": "The recorder only. The body is the raw MediaRecorder chunk (at most `max_chunk_bytes`, 512 KiB). Chunks are accepted in order only and each is fsynced before the reply; a re-send of a stored seq with the same bytes (sha256) is acknowledged again with `duplicate: true`. The first chunk must start with a WebM (EBML) or MP4 (ftyp) signature; the hive never decodes audio.", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Recording id from the start response" + }, + { + "name": "seq", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "minimum": 0 + }, + "description": "Chunk number, from 0" + } + ], + "requestBody": { + "required": true, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "responses": { + "200": { + "description": "Stored or recognised", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "seq": { + "type": "integer" + }, + "next_seq": { + "type": "integer" + }, + "bytes": { + "type": "integer" + }, + "duplicate": { + "type": "boolean" + } + } + } + } + } + }, + "400": { + "description": "Bad seq", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the recorder, or not owner or read-write", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown recording", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Out of order (expected_seq), different bytes for a stored seq, or already ended", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "expected_seq": { + "type": "integer", + "description": "Next seq the hive accepts (out-of-order chunk)" + }, + "active_id": { + "type": "string", + "description": "The user's recording already in progress" + } + } + } + } + } + }, + "413": { + "description": "Chunk over 512 KiB, or past inbox.recording.max_bytes / max_seconds", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "415": { + "description": "First chunk is not WebM or MP4", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "507": { + "description": "inbox.recording.quota_bytes used up", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/recordings/{id}/finalize": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Finish a recording", + "description": "The recorder only. Concatenates the chunks in order into audio.webm or audio.mp4, writes a pending Knowledge fact in the personal layer and lists the item in the Inbox. `duration_ms` is the recorder's own count (pauses excluded), clamped to the recording's span and max_seconds. Audited as `inbox_recording_finalize`.", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Recording id from the start response" + } + ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "duration_ms": { + "type": "integer" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Finalized", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "state": { + "type": "string" + }, + "title": { + "type": "string" + }, + "duration_ms": { + "type": "integer" + }, + "bytes": { + "type": "integer" + }, + "chunks": { + "type": "integer" + }, + "transcribes": { + "type": "boolean" + } + } + } + } + } + }, + "400": { + "description": "No chunk was received (the recording is removed)", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the recorder", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown recording", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Already ended", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/recordings/{id}": { + "delete": { + "tags": [ + "Knowledge" + ], + "summary": "Abort a recording", + "description": "The recorder, or an owner. Deletes an in-progress recording and its chunks. Audited as `inbox_recording_abort`.", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Recording id from the start response" + } + ], + "responses": { + "200": { + "description": "Aborted", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "aborted": { + "type": "boolean" + } + } + } + } + } + }, + "403": { + "description": "Not the recorder or an owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown recording", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Already ended", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/recordings/{id}/audio": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Play a finished recording", + "description": "Owner or read-write. Serves the recording from this hive's disk on the dashboard origin with Range support, `Cache-Control: no-store` and `Cross-Origin-Resource-Policy: same-origin`. Opening a playback is audited as `inbox_recording_audio`; seeks are not. The audio is deleted after inbox.recording.retention_days (default 5); the transcript and the Knowledge fact stay.", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Recording id from the start response" + }, + { + "name": "Range", + "in": "header", + "required": false, + "schema": { + "type": "string" + }, + "description": "A byte range (`bytes=start-end`)" + } + ], + "responses": { + "200": { + "description": "The whole recording", + "content": { + "audio/webm": { + "schema": { + "type": "string", + "format": "binary" + } + }, + "audio/mp4": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "206": { + "description": "The requested byte range", + "content": { + "audio/webm": { + "schema": { + "type": "string", + "format": "binary" + } + }, + "audio/mp4": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "403": { + "description": "Not owner or read-write", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown or unfinished recording", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "410": { + "description": "Audio expired after retention", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/api/inbox/transcriptions": { "get": { "tags": [ diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index b113864cff..73e1bf39f9 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -17,6 +17,10 @@ import ( // inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. const inboxSweepTimeout = 2 * time.Minute +// inboxRecordingSweepInterval is how often idle recordings are finalized and +// audio past inbox.recording.retention_days is deleted. +const inboxRecordingSweepInterval = 10 * time.Minute + // bootInbox builds the Inbox service. Sources are the configured knowledge // connectors whose type is a voice source (inbox.sources, default // plaud/wisprflow); with none configured the service still exists and lists @@ -56,7 +60,15 @@ func (b *boot) bootInbox(baseDir string) { } svc.EnableUploads(uploads) b.startInboxUploadRetention(uploads) + // Dashboard recordings (● Record): audio under connector-state/recordings + // until inbox.recording.retention_days, facts in the personal layer. + svc.EnableRecordings(&inbox.RecordingSource{ + Store: inbox.NewRecordingStore(filepath.Join(baseDir, "connector-state", "recordings")), + FactDir: filepath.Join(baseDir, "connectors", inbox.RecordingLayer), + RetentionDays: func() int { return svc.Config().Recording.EffectiveRetentionDays() }, + }) b.inboxSvc = svc + b.startInboxRecordingSweep(svc) b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) if tx := b.cfg.Inbox.Transcription; tx.Enabled { // Transcription reaches the provider from the hive process itself, @@ -215,3 +227,36 @@ func (b *boot) inboxAfterSync(st connector.Status) { return } } + +// startInboxRecordingSweep runs the recording sweep at boot and then every +// inboxRecordingSweepInterval until the boot context ends. Its changes are +// logged and audited with the inbox.RecordingSweepActor actor. +func (b *boot) startInboxRecordingSweep(svc *inbox.Service) { + ctx := b.ctx + if ctx == nil { + ctx = context.Background() + } + opts := inbox.RecordingOptions{Actor: inbox.RecordingSweepActor, Audit: func(a inbox.RecordingAudit) { + if b.dashSrv != nil { + b.dashSrv.AuditLog(a.Actor, a.Action, a.Detail(), "") + } + }} + sweep := func() { + if _, err := svc.RecordingSweep(opts); err != nil { + b.logger.Warn("inbox recording sweep", "error", err) + } + } + go func() { + sweep() + ticker := time.NewTicker(inboxRecordingSweepInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sweep() + } + } + }() +} diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 4fbaffcc1d..f774fe6545 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -27,7 +27,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1772` | | `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1773` | | `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1774` | -| `GET` | `/api/github-owner-identity` | Owner only | The hive owner's GitHub identity for remote repos ([#18197](https://github.com/hivecommons/hive/issues/18197)): `connected`, `login`, `scope`, `connected_at`, `usable`, `reason`, `verified_at`, `scope_required` (`public_repo`), `private_repos_supported` (false) and `file` (the token file path on the spoke). Never the token | `pkg/dashboard/api.go:550` | -| `POST` | `/api/github-owner-identity/start` | Owner only | Start the consent: a GitHub device flow with the spoke's `oauth_client_id` asking for `public_repo`, separate from the scope-free login flow. Returns `user_code`, `verification_uri`, `expires_in`, `interval`, `scope` and the `flow_id` the poll must present. 502 when GitHub refused the device-code request | `pkg/dashboard/api.go:551` | -| `POST` | `/api/github-owner-identity/poll` | Owner only | Poll the consent with `{"flow_id"}`: `status` `pending` / `slow_down` (poll again), `complete` (with `login` and the identity) or `error`. On completion the granted scope must cover `public_repo`, the token must validate and the account must be the logged-in owner's; the identity is then stored on the spoke (`/data/secrets/github-owner-token`, `HIVE_GITHUB_OWNER_TOKEN_FILE` overrides) and the reference-repo policies re-resolve. 400 without a live flow or with the wrong `flow_id`; audited as `github_owner_identity_connected` / `_rejected` | `pkg/dashboard/api.go:552` | -| `DELETE` | `/api/github-owner-identity` | Owner only | Disconnect: removes the token file, drops every write-tier reference repo to read-only on the next pass, returns the previous `login` and a `note` that GitHub-side revocation is done under Settings → Applications on github.com; audited as `github_owner_identity_disconnected` | `pkg/dashboard/api.go:553` | +| `GET` | `/api/github-owner-identity` | Owner only | The hive owner's GitHub identity for remote repos ([#18197](https://github.com/hivecommons/hive/issues/18197)): `connected`, `login`, `scope`, `connected_at`, `usable`, `reason`, `verified_at`, `scope_required` (`public_repo`), `private_repos_supported` (false) and `file` (the token file path on the spoke). Never the token | `pkg/dashboard/api.go:551` | +| `POST` | `/api/github-owner-identity/start` | Owner only | Start the consent: a GitHub device flow with the spoke's `oauth_client_id` asking for `public_repo`, separate from the scope-free login flow. Returns `user_code`, `verification_uri`, `expires_in`, `interval`, `scope` and the `flow_id` the poll must present. 502 when GitHub refused the device-code request | `pkg/dashboard/api.go:552` | +| `POST` | `/api/github-owner-identity/poll` | Owner only | Poll the consent with `{"flow_id"}`: `status` `pending` / `slow_down` (poll again), `complete` (with `login` and the identity) or `error`. On completion the granted scope must cover `public_repo`, the token must validate and the account must be the logged-in owner's; the identity is then stored on the spoke (`/data/secrets/github-owner-token`, `HIVE_GITHUB_OWNER_TOKEN_FILE` overrides) and the reference-repo policies re-resolve. 400 without a live flow or with the wrong `flow_id`; audited as `github_owner_identity_connected` / `_rejected` | `pkg/dashboard/api.go:553` | +| `DELETE` | `/api/github-owner-identity` | Owner only | Disconnect: removes the token file, drops every write-tier reference repo to read-only on the next pass, returns the previous `login` and a `note` that GitHub-side revocation is done under Settings → Applications on github.com; audited as `github_owner_identity_disconnected` | `pkg/dashboard/api.go:554` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index e9c42d4ec1..2f1fb3ebd4 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -101,8 +101,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:215` | | `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:522` | | `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:523` | -| `GET` | `/api/config/reference-repos` | Owner only | `project.reference_repos` ([#18136](https://github.com/hivecommons/hive/issues/18136)): `reference_repos` (`{repo, write, write_token_file}` plus `derived_from` on a derived entry and, on every write entry, `credential_state`, `credential_reason` and `credential_login`, the GitHub account the writes appear under ([#18197](https://github.com/hivecommons/hive/issues/18197)); always an array) and `write_kinds` | `pkg/dashboard/api.go:546` | -| `PUT` | `/api/config/reference-repos` | Owner only | Replace `project.reference_repos`; `{"reference_repos": []}` clears it. Refused with a 400 and nothing changed when the config loader would reject the list (bare name, repo also in `project.repos`, duplicate, unknown write kind, token file without write) or when a write entry has no credential boot would accept — the same message boot exits 22 with — or when an entry carries a `write_token_file` the config file does not already hold for that repo (the path is operator-only; a dashboard write opt-in writes as the hive owner's connected GitHub account). Entries with `derived_from` ([#18139](https://github.com/hivecommons/hive/issues/18139)) are the request issue's grant: echo them back as served; changing one's write set, leaving one out, sending one the hive did not derive or adding a hand-written entry for the same repo is a 409 naming the issue to change instead ([#18161](https://github.com/hivecommons/hive/issues/18161)). They are never stored: the saved list holds the hand-written entries and the derived set is merged back in memory after the save. 500 when the file could not be written (proxy restored, nothing changed). On success the GitHub proxy's reference policies are replaced without a restart; audited as `config_reference_repos` | `pkg/dashboard/api.go:547` | +| `GET` | `/api/config/reference-repos` | Owner only | `project.reference_repos` ([#18136](https://github.com/hivecommons/hive/issues/18136)): `reference_repos` (`{repo, write, write_token_file}` plus `derived_from` on a derived entry and, on every write entry, `credential_state`, `credential_reason` and `credential_login`, the GitHub account the writes appear under ([#18197](https://github.com/hivecommons/hive/issues/18197)); always an array) and `write_kinds` | `pkg/dashboard/api.go:547` | +| `PUT` | `/api/config/reference-repos` | Owner only | Replace `project.reference_repos`; `{"reference_repos": []}` clears it. Refused with a 400 and nothing changed when the config loader would reject the list (bare name, repo also in `project.repos`, duplicate, unknown write kind, token file without write) or when a write entry has no credential boot would accept — the same message boot exits 22 with — or when an entry carries a `write_token_file` the config file does not already hold for that repo (the path is operator-only; a dashboard write opt-in writes as the hive owner's connected GitHub account). Entries with `derived_from` ([#18139](https://github.com/hivecommons/hive/issues/18139)) are the request issue's grant: echo them back as served; changing one's write set, leaving one out, sending one the hive did not derive or adding a hand-written entry for the same repo is a 409 naming the issue to change instead ([#18161](https://github.com/hivecommons/hive/issues/18161)). They are never stored: the saved list holds the hand-written entries and the derived set is merged back in memory after the save. 500 when the file could not be written (proxy restored, nothing changed). On success the GitHub proxy's reference policies are replaced without a restart; audited as `config_reference_repos` | `pkg/dashboard/api.go:548` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:218` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:221` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:227` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 630586a337..7c1782c20a 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -77,6 +77,11 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/inbox/{source}/{id}/audio` | Owner or read-write | Mint a fresh short-lived audio/video link for one item (`{url, expires_at}`; Plaud: presigned, 24h; Zoom: audio-only or video `download_url` with a one-hour download token; Webex: the recording's temporary direct audio (else MP4) download link, valid 3 hours; Teams and Google Meet: 404, no browser-playable link; Slack huddles: 404, Slack keeps no huddle audio; Otter: the conversation's MP3 download link (`GET /conversations/{id}/audio`), minted per click; Granola: 404, Granola keeps no meeting audio; Limitless: 404, Pendant audio is served only behind the API key; Fathom: the signed url of a recording download (`POST /recordings/{id}/download`, audio-only recordings complete at once, video is polled), valid about 24 hours; Fireflies: the transcript's hashed `audio_url` (else `video_url`), valid 24 hours, Pro plan or higher); never stored by the hive, replied with `Cache-Control: no-store`, audited as `inbox_audio` (403 role, 404 unknown item or no audio, 502 source unreachable) | `pkg/dashboard/api_inbox.go:30` | | `GET` | `/api/inbox/{source}/{id}/audio/stream` | Owner or read-write | Stream one item's audio from the dashboard origin, for the Inbox player (the CSP refuses cross-origin media). The hive mints the link as above, fetches it server-side and copies the bytes with the media `Content-Type` (from storage, else the link's extension); a single `Range: bytes=…` is passed through (206 + `Content-Range`; 200 whole file when storage ignores it; 416 passed through). The link stays in memory for one playback's Range requests (10 min, never past expiry), is never logged, returned or stored, and is re-minted once when storage refuses it; nothing is written to disk. `Cache-Control: no-store`; audited as `inbox_audio_stream` once per minted link (403 role, 404 unknown item or no audio, 413 over the 2 GiB cap, 502 source or storage unreachable, or link refused/expired) | `pkg/dashboard/api_inbox.go:31` | | `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv and tsv become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:26` | +| `POST` | `/api/inbox/recordings` | Owner or read-write | Start a dashboard recording (● Record): `{title, mime: audio/webm\|audio/mp4, consent_version, includes_tab_audio}`. `consent_version` must name the current consent notice; it is stored with the user in the recording's metadata. One recording per user at a time. 201 `{id, max_chunk_bytes, max_seconds, retention_days, transcribes}`; audited as `inbox_recording_start` (400 consent missing or stale, 403 role, 409 `{active_id}` already recording, 412 `inbox.recording.enabled` off, 415 other media types, 507 `inbox.recording.quota_bytes` used up); see [inbox-recording.md](../inbox-recording.md) | `pkg/dashboard/api_inbox_recordings.go:20` | +| `POST` | `/api/inbox/recordings/{id}/chunks` | The recorder (owner or read-write) | Store one MediaRecorder chunk `?seq=N`, the raw bytes as the body (at most `max_chunk_bytes`, 512 KiB). In order only; each part is fsynced before the 200 `{id, seq, next_seq, bytes, duplicate?}`. The first chunk must start with a WebM or MP4 signature; the hive never decodes audio. A re-send with the same bytes is acknowledged again (`duplicate: true`) (400 bad seq, 403 someone else's, 409 `{expected_seq}` gap, other bytes for a stored seq, or already ended, 413 chunk over 512 KiB or past `max_bytes` / `max_seconds`, 415 not WebM/MP4, 507 quota) | `pkg/dashboard/api_inbox_recordings.go:21` | +| `POST` | `/api/inbox/recordings/{id}/finalize` | The recorder | End a recording: `{duration_ms?}` (the recorder's count, pauses excluded). The chunks are concatenated in order into `audio.webm` or `audio.mp4`, a pending Knowledge fact (personal layer) is written and the item appears in the Inbox. 200 `{id, state, title, duration_ms, bytes, chunks, transcribes}`; audited as `inbox_recording_finalize` (400 no chunk received, 403, 404, 409 already ended) | `pkg/dashboard/api_inbox_recordings.go:22` | +| `DELETE` | `/api/inbox/recordings/{id}` | The recorder, or an owner | Abort an in-progress recording and delete its chunks; audited as `inbox_recording_abort` (403, 404, 409 already ended) | `pkg/dashboard/api_inbox_recordings.go:23` | +| `GET` | `/api/inbox/recordings/{id}/audio` | Owner or read-write | Play a finished recording from this hive's disk on the dashboard origin: `http.ServeContent` (Range, 206), `Content-Type` audio/webm or audio/mp4, `Cache-Control: no-store`, `Cross-Origin-Resource-Policy: same-origin`. Opening a playback is audited as `inbox_recording_audio` (seeks are not). `GET /api/inbox/hive-recording/{id}/audio/stream` serves the same file (404 unknown or not finished, 410 audio expired after `inbox.recording.retention_days`) | `pkg/dashboard/api_inbox_recordings.go:24` | | `GET` | `/api/meetings/settings` | Owner only | Meetings settings: `enabled`, `enabled_env_override`, `running`, `vexa_url`, `vexa_key_set`, `discord_token_set`, `transcribe_token_set` (secrets are never returned) and `effective` values of the other `meetings.*` keys ([env vars](../env-vars.md#meetings-quorum)) | `pkg/dashboard/api_meetings.go:123` | | `POST` | `/api/meetings/settings` | Owner only | Update `meetings.enabled` and `meetings.vexa.url`, and write the Vexa tenant key, Discord bot token and transcription service token to keystore-sealed secret files (`""` clears one): `{enabled?, vexa_url?, vexa_api_key?, discord_token?, transcribe_token?}`. Returns the settings with `restart_required`; 400 on an invalid value. Audited as `meetings_settings` | `pkg/dashboard/api_meetings.go:124` | | `POST` | `/api/meetings` | Owner only | Send hive to a meeting: `{url}` for Google Meet, Teams or Zoom, or `{platform: "discord", guild_id, channel_id}`, plus `title?` and `voice?`. Returns the meeting record; 400 invalid request, 502 Quorum refused, 503 meetings off. Audited as `meeting_join` | `pkg/dashboard/api_meetings.go:125` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index be1cb38008..eb040a8909 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:341`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:342` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:343` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:544`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:545`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:333`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:345`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:346` | diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 6157ec2603..a00b938f27 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -176,6 +176,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Inbox and voice sources](inbox-voice-sources.md) — triage Plaud/Wispr Flow recordings into spektacular RFCs, dibs ideas, or knowledge only; hosted-hive Plaud sign-in. - [Add files and links to the Inbox](inbox-uploads.md) — drop or paste md, txt, json, jsonl, yaml, csv and tsv files onto the Inbox: content sniffing, limits, retention and privacy. - [Inbox transcription](inbox-transcription.md) — spoke-local Deepgram transcription of recordings without text, with the owner's own key, budget and consent gate. +- [Recording into the Inbox](inbox-recording.md) — record a meeting or note from the dashboard (● Record): consent notice before every recording, chunked upload, 5-day audio retention, browser support. - [Meetings](meetings.md) — send hive into Google Meet, Teams, Zoom or Discord meetings to take notes, ask questions and run research; consent, setup and settings. - [Knowledge curator](knowledge-curator.md) — configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) — interpret confidence badges and score explanations. diff --git a/src/docs/inbox-recording.md b/src/docs/inbox-recording.md new file mode 100644 index 0000000000..6fcd33926f --- /dev/null +++ b/src/docs/inbox-recording.md @@ -0,0 +1,102 @@ +# Recording into the Inbox + +**● Record** in the [Inbox](inbox-voice-sources.md) records a meeting or a +note straight from the dashboard, without a recorder device or a meeting +connector. The audio is uploaded to this hive while you record, the finished +recording appears in the Inbox like any other item, and it can be +transcribed, read and shared (Share → Hive: RFC, spek, Knowledge, DIBS). + +## Who can record + +Owners and read-write members. Read-only members do not see ● Record and the +API answers them 403. Each member may have one recording in progress at a +time. An owner can abort anyone's recording; everyone else can only stop or +abort their own. + +## Consent + +The dashboard shows a consent notice before **every** recording, and a red +**● Recording** indicator stays on screen for the whole recording. Starting a +recording stores the notice version you acknowledged and your user with the +recording (`consent_version`, `recorded_by`, `consent_at` in its metadata, and +in the Knowledge fact). + +Recording laws differ between countries, states and workplaces: some allow a +recording when one participant agrees, others need everyone's agreement, and +many require that people are told before the recording starts. **Making sure +a recording is lawful, and that everyone recorded knows about it, is the +responsibility of the person who records.** The hive does not decide this for +you. Good practice: + +- say at the start that you are recording and why, and offer to stop; +- when you include a browser tab (a video meeting), the remote participants + are recorded too: tell them, or use the meeting platform's own recording + notice; +- do not record people who have not been told. + +## Retention + +| What | Kept | +| --- | --- | +| Raw audio (`audio.webm` / `audio.mp4`) | `inbox.recording.retention_days` after the recording ends (default **5 days**), then deleted | +| Transcript (machine transcript on this hive) | kept | +| Knowledge fact (personal layer, never published to the hub) | kept | +| Recording metadata (title, who, when, duration, consent version) | kept | + +After the audio is deleted the item shows **audio expired (kept 5 days)** +(with the configured number of days) and has no play control; the transcript +and the fact stay readable and shareable. The sweep that deletes audio runs at +boot and every 10 minutes, and logs and audits every deletion +(`inbox_recording_audio_expired`, actor `inbox-recording-sweep`). + +## Configuration + +```yaml +inbox: + recording: + enabled: false # show ● Record (owner and read-write only) + retention_days: 5 # raw audio is deleted this long after the recording ends + max_seconds: 10800 # 3 hours per recording (wall clock, pauses included) + max_bytes: 209715200 # 200 MiB per recording + quota_bytes: 2147483648 # 2 GiB of recording audio on this hive at once + idle_finalize_minutes: 120 # a recording whose browser went silent this long is finished automatically +``` + +`max_bytes` must not exceed `quota_bytes`. At the dashboard's 32 kbit/s a +3-hour recording is about 45 MB, so the defaults leave room for long meetings +and many recordings at once. + +## How a recording is stored + +- The browser records with `MediaRecorder` and uploads one chunk every 10 + seconds (`POST /api/inbox/recordings/{id}/chunks?seq=N`), each at most + 512 KiB, so it fits the default 1 MiB ingress body limit and the + dashboard's 30 second request timeout. +- Chunks are accepted in order only. Each one is written and fsynced as its + own part file before the hive acknowledges it; an out-of-order chunk gets + 409 with the expected `seq`, and a chunk re-sent with the same bytes is + acknowledged again without being stored twice. That makes a resumed + upload safe after a network drop or a reload. +- Stopping (finalize) concatenates the parts in order into one file under + `/connector-state/recordings//` (0700 directory, 0600 + files) and writes a pending Knowledge fact. +- The hive only checks the container signature of the first chunk (WebM or + MP4) and never decodes audio itself; the transcription provider does (the + sandboxed whisper subprocess by default). +- A recording whose browser stops sending chunks for + `idle_finalize_minutes` is finished automatically with what arrived; one + with no audio at all is removed. + +Playback is served from this hive's disk on the dashboard origin +(`GET /api/inbox/recordings/{id}/audio`, with Range for seeking and +`Cache-Control: no-store`). See the [API reference](api-reference/knowledge.md). + +### Safari and MP4 + +Safari records fragmented MP4: the first chunk carries the `ftyp` and `moov` +boxes, every later chunk a `moof` + `mdat` fragment. Concatenating the chunks +of one recorder in order gives a valid fragmented MP4, so the hive uses one +recorder for the whole recording (pause and resume included) and stores one +file. On finalize the hive walks the top-level MP4 boxes of the result and +logs a warning if they do not chain to the end of the file; the file is kept +either way. diff --git a/src/pkg/config/inbox_config.go b/src/pkg/config/inbox_config.go index 712a2fada5..d818e6474d 100644 --- a/src/pkg/config/inbox_config.go +++ b/src/pkg/config/inbox_config.go @@ -36,6 +36,9 @@ type InboxConfig struct { // Uploads bounds files and links dropped onto the Inbox (see // inbox_uploads.go). Uploads InboxUploads `yaml:"uploads,omitempty" json:"uploads,omitzero"` + // Recording is recording straight from the dashboard into the Inbox + // (see inbox_recording.go). + Recording InboxRecording `yaml:"recording,omitempty" json:"recording,omitzero"` } // InboxRule routes items whose source and title/tag match. @@ -118,6 +121,9 @@ func ValidateInbox(c InboxConfig) error { if err := ValidateInboxUploads(c.Uploads); err != nil { return err } + if err := ValidateInboxRecording(c.Recording); err != nil { + return err + } for i, r := range c.Rules { switch r.Destination { case InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss: diff --git a/src/pkg/config/inbox_recording.go b/src/pkg/config/inbox_recording.go new file mode 100644 index 0000000000..2519fcdd32 --- /dev/null +++ b/src/pkg/config/inbox_recording.go @@ -0,0 +1,115 @@ +package config + +import ( + "fmt" + "time" +) + +// Inbox recording defaults (`inbox.recording`): recording a meeting or a +// note straight from the dashboard into the Inbox. Every value is a named +// default the owner can override in hive.yaml. +const ( + // DefaultInboxRecordingEnabled shows the ● Record control. Recording is + // still limited to owner and read-write sessions and needs the consent + // notice acknowledged before every recording. + DefaultInboxRecordingEnabled = false + // DefaultInboxRecordingRetentionDays is how long the raw audio is kept + // after a recording ends; the transcript and the Knowledge fact stay. + DefaultInboxRecordingRetentionDays = 5 + // DefaultInboxRecordingMaxSeconds caps one recording (3 hours). + DefaultInboxRecordingMaxSeconds = 3 * 60 * 60 + // DefaultInboxRecordingMaxBytes caps one recording's audio (200 MiB). + DefaultInboxRecordingMaxBytes int64 = 200 << 20 + // DefaultInboxRecordingQuotaBytes caps all recording audio kept on this + // hive at once (2 GiB). + DefaultInboxRecordingQuotaBytes int64 = 2 << 30 + // DefaultInboxRecordingIdleFinalizeMinutes finalizes a recording whose + // browser stopped sending chunks this long ago (2 hours). + DefaultInboxRecordingIdleFinalizeMinutes = 120 +) + +// InboxRecording is `inbox.recording`. +type InboxRecording struct { + // Enabled shows ● Record in the Inbox; nil uses + // DefaultInboxRecordingEnabled. + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // RetentionDays keeps the raw audio this many days after the recording + // ends, then deletes it (0 uses the default). + RetentionDays int `yaml:"retention_days,omitempty" json:"retention_days,omitempty"` + // MaxSeconds caps one recording's length (0 uses the default). + MaxSeconds int `yaml:"max_seconds,omitempty" json:"max_seconds,omitempty"` + // MaxBytes caps one recording's audio size (0 uses the default). + MaxBytes int64 `yaml:"max_bytes,omitempty" json:"max_bytes,omitempty"` + // QuotaBytes caps the audio of all recordings kept at once (0 uses the + // default). + QuotaBytes int64 `yaml:"quota_bytes,omitempty" json:"quota_bytes,omitempty"` + // IdleFinalizeMinutes finalizes a recording that received no chunk for + // this long (0 uses the default). + IdleFinalizeMinutes int `yaml:"idle_finalize_minutes,omitempty" json:"idle_finalize_minutes,omitempty"` +} + +// EffectiveEnabled reports whether ● Record is offered. +func (r InboxRecording) EffectiveEnabled() bool { + if r.Enabled == nil { + return DefaultInboxRecordingEnabled + } + return *r.Enabled +} + +// EffectiveRetentionDays returns the audio retention in days. +func (r InboxRecording) EffectiveRetentionDays() int { + if r.RetentionDays > 0 { + return r.RetentionDays + } + return DefaultInboxRecordingRetentionDays +} + +// EffectiveRetention returns the audio retention as a duration. +func (r InboxRecording) EffectiveRetention() time.Duration { + return time.Duration(r.EffectiveRetentionDays()) * 24 * time.Hour +} + +// EffectiveMaxSeconds returns the longest recording allowed, in seconds. +func (r InboxRecording) EffectiveMaxSeconds() int { + if r.MaxSeconds > 0 { + return r.MaxSeconds + } + return DefaultInboxRecordingMaxSeconds +} + +// EffectiveMaxBytes returns the per-recording audio cap. +func (r InboxRecording) EffectiveMaxBytes() int64 { + if r.MaxBytes > 0 { + return r.MaxBytes + } + return DefaultInboxRecordingMaxBytes +} + +// EffectiveQuotaBytes returns the hive-wide recording audio cap. +func (r InboxRecording) EffectiveQuotaBytes() int64 { + if r.QuotaBytes > 0 { + return r.QuotaBytes + } + return DefaultInboxRecordingQuotaBytes +} + +// EffectiveIdleFinalize returns how long a silent recording stays open. +func (r InboxRecording) EffectiveIdleFinalize() time.Duration { + m := r.IdleFinalizeMinutes + if m <= 0 { + m = DefaultInboxRecordingIdleFinalizeMinutes + } + return time.Duration(m) * time.Minute +} + +// ValidateInboxRecording rejects negative limits and a per-recording cap +// larger than the hive-wide quota. +func ValidateInboxRecording(r InboxRecording) error { + if r.RetentionDays < 0 || r.MaxSeconds < 0 || r.MaxBytes < 0 || r.QuotaBytes < 0 || r.IdleFinalizeMinutes < 0 { + return fmt.Errorf("inbox.recording limits must not be negative") + } + if r.EffectiveMaxBytes() > r.EffectiveQuotaBytes() { + return fmt.Errorf("inbox.recording.max_bytes (%d) must not exceed inbox.recording.quota_bytes (%d)", r.EffectiveMaxBytes(), r.EffectiveQuotaBytes()) + } + return nil +} diff --git a/src/pkg/config/inbox_recording_test.go b/src/pkg/config/inbox_recording_test.go new file mode 100644 index 0000000000..6a2c3efe63 --- /dev/null +++ b/src/pkg/config/inbox_recording_test.go @@ -0,0 +1,47 @@ +package config + +import ( + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestInboxRecordingDefaults(t *testing.T) { + var r InboxRecording + if r.EffectiveEnabled() != DefaultInboxRecordingEnabled { + t.Fatal("enabled default") + } + if r.EffectiveRetentionDays() != 5 || r.EffectiveRetention() != 5*24*time.Hour { + t.Fatalf("retention = %d", r.EffectiveRetentionDays()) + } + if r.EffectiveMaxSeconds() != 3*60*60 || r.EffectiveMaxBytes() != 200<<20 || r.EffectiveQuotaBytes() != 2<<30 || r.EffectiveIdleFinalize() != 2*time.Hour { + t.Fatalf("caps = %d %d %d %v", r.EffectiveMaxSeconds(), r.EffectiveMaxBytes(), r.EffectiveQuotaBytes(), r.EffectiveIdleFinalize()) + } + if err := ValidateInboxRecording(r); err != nil { + t.Fatal(err) + } +} + +func TestInboxRecordingYAMLAndValidation(t *testing.T) { + var c InboxConfig + src := "recording:\n enabled: true\n retention_days: 7\n max_seconds: 600\n max_bytes: 1000\n quota_bytes: 5000\n idle_finalize_minutes: 30\n" + if err := yaml.Unmarshal([]byte(src), &c); err != nil { + t.Fatal(err) + } + r := c.Recording + if !r.EffectiveEnabled() || r.EffectiveRetentionDays() != 7 || r.EffectiveMaxSeconds() != 600 || r.EffectiveMaxBytes() != 1000 || r.EffectiveQuotaBytes() != 5000 || r.EffectiveIdleFinalize() != 30*time.Minute { + t.Fatalf("parsed = %+v", r) + } + if err := ValidateInbox(c); err != nil { + t.Fatal(err) + } + c.Recording.MaxBytes = 6000 + if err := ValidateInbox(c); err == nil { + t.Fatal("max_bytes over quota_bytes accepted") + } + c.Recording = InboxRecording{RetentionDays: -1} + if err := ValidateInbox(c); err == nil { + t.Fatal("negative retention accepted") + } +} diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index d3c7f33d12..5f8ea791aa 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -535,6 +535,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.registerKnowledgeConnectorRoutes() s.registerKnowledgeConnectorAuthRoutes() s.registerInboxRoutes() + s.registerInboxRecordingRoutes() s.registerMeetingsRoutes() s.registerMeetingsScheduleRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go index a1a3543a3f..986f2d4c51 100644 --- a/src/pkg/dashboard/api_inbox.go +++ b/src/pkg/dashboard/api_inbox.go @@ -167,6 +167,15 @@ func (s *Server) handleInboxAudioStream(w http.ResponseWriter, r *http.Request) if !ok { return } + if a, local, err := svc.OpenLocalAudio(r.Context(), it); local { + // Audio kept on this hive (dashboard recordings) is served from disk. + if err != nil { + writeInboxLocalAudioError(w, svc, err) + return + } + s.serveLocalAudio(w, r, a, "inbox_audio_stream", auditDetail("source", it.Source, "item", it.ID)) + return + } w.Header().Set("Cache-Control", "no-store") st, err := svc.OpenAudioStream(r.Context(), it, r.Header.Get("Range")) switch { diff --git a/src/pkg/dashboard/api_inbox_recordings.go b/src/pkg/dashboard/api_inbox_recordings.go new file mode 100644 index 0000000000..f1b67e4636 --- /dev/null +++ b/src/pkg/dashboard/api_inbox_recordings.go @@ -0,0 +1,290 @@ +package dashboard + +import ( + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/inbox" +) + +// Recording straight into the Inbox (● Record). The browser streams the +// MediaRecorder output as ordered chunks while it records; see +// pkg/inbox/recordings.go for storage and docs/inbox-recording.md for the +// user guide. Every route needs an owner or read-write session; read-only +// sessions get 403. +func (s *Server) registerInboxRecordingRoutes() { + s.mux.HandleFunc("POST /api/inbox/recordings", s.handleInboxRecordingStart) + s.mux.HandleFunc("POST /api/inbox/recordings/{id}/chunks", s.handleInboxRecordingChunk) + s.mux.HandleFunc("POST /api/inbox/recordings/{id}/finalize", s.handleInboxRecordingFinalize) + s.mux.HandleFunc("DELETE /api/inbox/recordings/{id}", s.handleInboxRecordingAbort) + s.mux.HandleFunc("GET /api/inbox/recordings/{id}/audio", s.handleInboxRecordingAudio) +} + +// inboxRecordingStartRequest is POST /api/inbox/recordings. +type inboxRecordingStartRequest struct { + Title string `json:"title"` + MIME string `json:"mime"` + ConsentVersion string `json:"consent_version"` + IncludesTabAudio bool `json:"includes_tab_audio"` +} + +// InboxRecordingStartResponse is what a start returns: the id and the +// limits the recorder must respect. +type InboxRecordingStartResponse struct { + ID string `json:"id"` + MaxChunkBytes int `json:"max_chunk_bytes"` + MaxSeconds int `json:"max_seconds"` + RetentionDays int `json:"retention_days"` + // Transcribes reports whether the hive transcribes the recording when + // it ends (transcription enabled with consent). + Transcribes bool `json:"transcribes"` +} + +// InboxRecordingChunkResponse is POST /api/inbox/recordings/{id}/chunks. +type InboxRecordingChunkResponse struct { + ID string `json:"id"` + inbox.RecordingChunk +} + +// InboxRecordingFinalizeResponse is POST /api/inbox/recordings/{id}/finalize. +type InboxRecordingFinalizeResponse struct { + ID string `json:"id"` + State string `json:"state"` + Title string `json:"title"` + DurationMS int64 `json:"duration_ms"` + Bytes int64 `json:"bytes"` + Chunks int `json:"chunks"` + // Transcribes reports whether the hive transcribes it now. + Transcribes bool `json:"transcribes"` +} + +// inboxRecordingError is a recording error body; ExpectedSeq / ActiveID are +// set for an out-of-order chunk and an already-active recording. +type inboxRecordingError struct { + Error string `json:"error"` + ExpectedSeq *int `json:"expected_seq,omitempty"` + ActiveID string `json:"active_id,omitempty"` +} + +// inboxRecordingOptions audits as the requesting user. +func (s *Server) inboxRecordingOptions(r *http.Request) inbox.RecordingOptions { + return inbox.RecordingOptions{Actor: requestUser(r), Audit: func(a inbox.RecordingAudit) { + if s.audit != nil { + s.audit.Log(a.Actor, a.Action, auditDetail(a.Fields...), "") + } + }} +} + +// writeInboxRecordingError maps recording errors to statuses. +func writeInboxRecordingError(w http.ResponseWriter, err error) { + var gap *inbox.RecordingGapError + var active *inbox.RecordingActiveError + body := inboxRecordingError{Error: err.Error()} + code := http.StatusInternalServerError + switch { + case errors.As(err, &gap): + code, body.ExpectedSeq = http.StatusConflict, &gap.Expected + case errors.As(err, &active): + code, body.ActiveID = http.StatusConflict, active.ID + case errors.Is(err, inbox.ErrRecordingDisabled): + code = http.StatusPreconditionFailed + case errors.Is(err, inbox.ErrRecordingNotFound): + code = http.StatusNotFound + case errors.Is(err, inbox.ErrRecordingNotYours): + code = http.StatusForbidden + case errors.Is(err, inbox.ErrRecordingClosed), errors.Is(err, inbox.ErrRecordingMismatch): + code = http.StatusConflict + case errors.Is(err, inbox.ErrRecordingConsent), errors.Is(err, inbox.ErrRecordingBadRequest), errors.Is(err, inbox.ErrRecordingEmpty): + code = http.StatusBadRequest + case errors.Is(err, inbox.ErrRecordingMIME), errors.Is(err, inbox.ErrRecordingNotMedia): + code = http.StatusUnsupportedMediaType + case errors.Is(err, inbox.ErrRecordingTooLarge), errors.Is(err, inbox.ErrRecordingTooLong), errors.Is(err, inbox.ErrRecordingChunkSize): + code = http.StatusRequestEntityTooLarge + case errors.Is(err, inbox.ErrRecordingQuota): + code = http.StatusInsufficientStorage + case errors.Is(err, inbox.ErrRecordingExpired): + code = http.StatusGone + case errors.Is(err, inbox.ErrNoAudio): + code = http.StatusNotFound + default: + body.Error = "recording failed: " + err.Error() + } + jsonStatusResponse(w, code, body) +} + +// handleInboxRecordingStart (owner or read-write) starts a recording. The +// request must acknowledge the current consent notice; the notice version, +// the user and whether tab audio is included are stored with it. One +// recording per user may be in progress (409 with active_id). +func (s *Server) handleInboxRecordingStart(w http.ResponseWriter, r *http.Request) { + if !requireInboxPlaybackRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxRecordingStartRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + m, err := svc.StartRecording(inbox.RecordingStart{ + Title: sanitizeString(req.Title), MIME: req.MIME, ConsentVersion: req.ConsentVersion, + IncludesTabAudio: req.IncludesTabAudio, User: requestUser(r), Role: r.Header.Get("X-Hive-Role"), + }, s.inboxRecordingOptions(r)) + if err != nil { + writeInboxRecordingError(w, err) + return + } + st := svc.RecordingSettings() + w.Header().Set("Cache-Control", "no-store") + jsonStatusResponse(w, http.StatusCreated, InboxRecordingStartResponse{ + ID: m.ID, MaxChunkBytes: st.MaxChunkBytes, MaxSeconds: st.MaxSeconds, RetentionDays: st.RetentionDays, Transcribes: st.Transcribes, + }) +} + +// handleInboxRecordingChunk (the recorder) stores chunk ?seq=N, the raw +// MediaRecorder bytes as the body (at most max_chunk_bytes). Chunks are +// accepted in order only: a gap answers 409 with expected_seq, and a re-send +// of a stored chunk with the same bytes answers 200 again (duplicate: true). +func (s *Server) handleInboxRecordingChunk(w http.ResponseWriter, r *http.Request) { + if !requireInboxPlaybackRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + defer closeHTTPBody(r.Body) + seq, err := strconv.Atoi(r.URL.Query().Get("seq")) + if err != nil || seq < 0 { + jsonError(w, "seq must be a non-negative integer", http.StatusBadRequest) + return + } + body := http.MaxBytesReader(w, r.Body, inbox.RecordingMaxChunkBytes) + data, err := io.ReadAll(body) + if err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + jsonError(w, fmt.Sprintf("a chunk must be at most %d bytes", inbox.RecordingMaxChunkBytes), http.StatusRequestEntityTooLarge) + return + } + jsonError(w, "reading chunk: "+err.Error(), http.StatusBadRequest) + return + } + id := strings.TrimSpace(r.PathValue("id")) + res, err := svc.AppendRecordingChunk(id, requestUser(r), seq, data) + if err != nil { + writeInboxRecordingError(w, err) + return + } + w.Header().Set("Cache-Control", "no-store") + jsonResponse(w, InboxRecordingChunkResponse{ID: id, RecordingChunk: res}) +} + +// inboxRecordingFinalizeRequest is the optional finalize body. +type inboxRecordingFinalizeRequest struct { + // DurationMS is the recorder's own count, pauses excluded. + DurationMS int64 `json:"duration_ms"` +} + +// handleInboxRecordingFinalize (the recorder) ends a recording: the chunks +// are concatenated into one audio file and the item appears in the Inbox. +func (s *Server) handleInboxRecordingFinalize(w http.ResponseWriter, r *http.Request) { + if !requireInboxPlaybackRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxRecordingFinalizeRequest + if r.ContentLength != 0 { + if err := decodeBody(r, &req); err != nil && !errors.Is(err, io.EOF) { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + } + m, err := svc.FinalizeRecording(strings.TrimSpace(r.PathValue("id")), requestUser(r), req.DurationMS, s.inboxRecordingOptions(r)) + if err != nil { + writeInboxRecordingError(w, err) + return + } + jsonResponse(w, InboxRecordingFinalizeResponse{ + ID: m.ID, State: m.State, Title: m.Title, DurationMS: m.DurationMS, Bytes: m.Bytes, Chunks: m.Chunks, + Transcribes: svc.RecordingSettings().Transcribes, + }) +} + +// handleInboxRecordingAbort discards an in-progress recording: the +// recorder's own, or anyone's for an owner. +func (s *Server) handleInboxRecordingAbort(w http.ResponseWriter, r *http.Request) { + if !requireInboxPlaybackRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + asOwner := isOwnerRole(r.Header.Get("X-Hive-Role")) + m, err := svc.AbortRecording(strings.TrimSpace(r.PathValue("id")), requestUser(r), asOwner, s.inboxRecordingOptions(r)) + if err != nil { + writeInboxRecordingError(w, err) + return + } + jsonResponse(w, map[string]any{"id": m.ID, "aborted": true}) +} + +// handleInboxRecordingAudio (owner or read-write) serves a finished +// recording from this hive's disk, on the dashboard origin, with Range +// support (http.ServeContent) and no-store. After retention the audio is +// gone: 410. Opening a playback (no Range, or a range from byte 0) is +// audited as inbox_recording_audio; the player's seeks are not. +func (s *Server) handleInboxRecordingAudio(w http.ResponseWriter, r *http.Request) { + if !requireInboxPlaybackRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + id := strings.TrimSpace(r.PathValue("id")) + a, err := svc.OpenRecordingAudio(id) + if err != nil { + writeInboxLocalAudioError(w, svc, err) + return + } + s.serveLocalAudio(w, r, a, "inbox_recording_audio", auditDetail("recording", id)) +} + +// serveLocalAudio writes a hive-local audio file with Range support. +func (s *Server) serveLocalAudio(w http.ResponseWriter, r *http.Request, a inbox.LocalAudio, action, detail string) { + defer a.File.Close() + if rng := r.Header.Get("Range"); rng == "" || strings.HasPrefix(rng, "bytes=0-") { + s.auditFromRequest(r, action, detail, "") + } + h := w.Header() + h.Set("Cache-Control", "no-store") + h.Set("Content-Type", a.MIME) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Content-Disposition", "inline") + h.Set("Cross-Origin-Resource-Policy", "same-origin") + http.ServeContent(w, r, a.Name, a.ModTime, a.File) +} + +// writeInboxLocalAudioError answers a failed local audio open; expired +// audio is 410 with the retention named. +func writeInboxLocalAudioError(w http.ResponseWriter, svc *inbox.Service, err error) { + w.Header().Set("Cache-Control", "no-store") + if errors.Is(err, inbox.ErrRecordingExpired) { + days := svc.RecordingSettings().RetentionDays + jsonError(w, fmt.Sprintf("audio expired (kept %d days); the transcript and the Knowledge fact are kept", days), http.StatusGone) + return + } + writeInboxRecordingError(w, err) +} diff --git a/src/pkg/dashboard/api_inbox_recordings_test.go b/src/pkg/dashboard/api_inbox_recordings_test.go new file mode 100644 index 0000000000..cb541b640c --- /dev/null +++ b/src/pkg/dashboard/api_inbox_recordings_test.go @@ -0,0 +1,260 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" +) + +// recordingWebM is a first chunk that starts with the WebM (EBML) header. +var recordingWebM = append([]byte{0x1A, 0x45, 0xDF, 0xA3}, []byte("opus-frames-0123456789")...) + +// newRecordingTestServer wires an Inbox with recording enabled. +func newRecordingTestServer(t *testing.T, rec config.InboxRecording) (*Server, *inbox.Service, *inbox.RecordingSource) { + t.Helper() + s, deps := apiServer(t) + if rec.Enabled == nil { + on := true + rec.Enabled = &on + } + deps.Config.Inbox = config.InboxConfig{Recording: rec} + dir := t.TempDir() + svc := inbox.NewService(deps.Config.Inbox, inbox.NewStore(filepath.Join(dir, "_inbox.json")), nil, nil, nil) + svc.Run = func(fn func()) { fn() } + svc.EnableTranscription(inbox.NewTranscriptStore(filepath.Join(dir, "transcripts"))) + src := &inbox.RecordingSource{Store: inbox.NewRecordingStore(filepath.Join(dir, "recordings")), FactDir: filepath.Join(dir, "connectors", inbox.RecordingLayer)} + svc.EnableRecordings(src) + deps.Inbox = svc + return s, svc, src +} + +// doRecording sends one request as role (user @example.com). +func doRecording(s *Server, method, path, role string, body []byte, hdr map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + if role != "" { + req.Header.Set("X-Hive-Role", role) + req.Header.Set("X-Hive-User", role+"@example.com") + if role == config.RoleOwner { + req.Header.Set(ownerRoleVerifiedHeader, "true") + } + } + for k, v := range hdr { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + s.mux.ServeHTTP(rec, req) + return rec +} + +func recordingStartBody(t *testing.T, consent string) []byte { + t.Helper() + b, _ := json.Marshal(map[string]any{"title": "Weekly sync x", "mime": "audio/webm;codecs=opus", "consent_version": consent, "includes_tab_audio": false}) + return b +} + +func startTestRecording(t *testing.T, s *Server, role string) string { + t.Helper() + rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", role, recordingStartBody(t, inbox.RecordingConsentVersion), map[string]string{"Content-Type": "application/json"}) + if rec.Code != http.StatusCreated { + t.Fatalf("start as %s = %d %s", role, rec.Code, rec.Body) + } + var resp InboxRecordingStartResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.MaxChunkBytes != inbox.RecordingMaxChunkBytes || resp.MaxSeconds != config.DefaultInboxRecordingMaxSeconds || resp.RetentionDays != config.DefaultInboxRecordingRetentionDays { + t.Fatalf("start response = %+v", resp) + } + return resp.ID +} + +func chunkPath(id string, seq int) string { + return "/api/inbox/recordings/" + id + "/chunks?seq=" + strconv.Itoa(seq) +} + +func TestInboxRecordingRoleMatrix(t *testing.T) { + s, _, _ := newRecordingTestServer(t, config.InboxRecording{}) + body := recordingStartBody(t, inbox.RecordingConsentVersion) + for _, role := range []string{"", config.RoleRead} { + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", role, body, nil); rec.Code != http.StatusForbidden { + t.Fatalf("start as %q = %d, want 403", role, rec.Code) + } + } + // An owner header without the verified marker is not an owner. + req := httptest.NewRequest(http.MethodPost, "/api/inbox/recordings", bytes.NewReader(body)) + req.Header.Set("X-Hive-Role", config.RoleOwner) + rec := httptest.NewRecorder() + s.mux.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("unverified owner = %d, want 403", rec.Code) + } + id := startTestRecording(t, s, config.RoleReadWrite) + for _, tc := range []struct{ method, path string }{ + {http.MethodPost, chunkPath(id, 0)}, + {http.MethodPost, "/api/inbox/recordings/" + id + "/finalize"}, + {http.MethodDelete, "/api/inbox/recordings/" + id}, + {http.MethodGet, "/api/inbox/recordings/" + id + "/audio"}, + } { + if rec := doRecording(s, tc.method, tc.path, config.RoleRead, recordingWebM, nil); rec.Code != http.StatusForbidden { + t.Fatalf("%s %s as read-only = %d, want 403", tc.method, tc.path, rec.Code) + } + } + startTestRecording(t, s, config.RoleOwner) + if e := s.audit.Recent(1); len(e) != 1 || e[0].Action != "inbox_recording_start" || !strings.Contains(e[0].Detail, "consent_version="+inbox.RecordingConsentVersion) { + t.Fatalf("start not audited: %+v", e) + } +} + +func TestInboxRecordingDisabledAndConsent(t *testing.T) { + off := false + s, _, _ := newRecordingTestServer(t, config.InboxRecording{Enabled: &off}) + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", config.RoleReadWrite, recordingStartBody(t, inbox.RecordingConsentVersion), nil); rec.Code != http.StatusPreconditionFailed { + t.Fatalf("disabled start = %d, want 412", rec.Code) + } + s, _, _ = newRecordingTestServer(t, config.InboxRecording{}) + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", config.RoleReadWrite, recordingStartBody(t, ""), nil); rec.Code != http.StatusBadRequest { + t.Fatalf("start without consent = %d, want 400", rec.Code) + } + b, _ := json.Marshal(map[string]any{"mime": "audio/ogg", "consent_version": inbox.RecordingConsentVersion}) + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", config.RoleReadWrite, b, nil); rec.Code != http.StatusUnsupportedMediaType { + t.Fatalf("ogg start = %d, want 415", rec.Code) + } + id := startTestRecording(t, s, config.RoleReadWrite) + rec := doRecording(s, http.MethodPost, "/api/inbox/recordings", config.RoleReadWrite, recordingStartBody(t, inbox.RecordingConsentVersion), nil) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), `"active_id":"`+id+`"`) { + t.Fatalf("second start = %d %s, want 409 with active_id", rec.Code, rec.Body) + } +} + +func TestInboxRecordingChunksFinalizeAndAudio(t *testing.T) { + s, svc, src := newRecordingTestServer(t, config.InboxRecording{}) + id := startTestRecording(t, s, config.RoleReadWrite) + if m, _ := src.Store.Get(id); m.Title != "Weekly sync x" { + t.Fatalf("title not sanitized: %q", m.Title) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, []byte(""), nil); rec.Code != http.StatusUnsupportedMediaType { + t.Fatalf("non-media first chunk = %d, want 415", rec.Code) + } + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings/"+id+"/chunks?seq=x", config.RoleReadWrite, recordingWebM, nil); rec.Code != http.StatusBadRequest { + t.Fatalf("bad seq = %d, want 400", rec.Code) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, make([]byte, inbox.RecordingMaxChunkBytes+1), nil); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversize chunk = %d, want 413", rec.Code) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleOwner, recordingWebM, nil); rec.Code != http.StatusForbidden { + t.Fatalf("someone else's chunk = %d, want 403", rec.Code) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, recordingWebM, nil); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"next_seq":1`) { + t.Fatalf("chunk 0 = %d %s", rec.Code, rec.Body) + } + rec := doRecording(s, http.MethodPost, chunkPath(id, 2), config.RoleReadWrite, []byte("gap"), nil) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), `"expected_seq":1`) { + t.Fatalf("gap = %d %s, want 409 expected_seq 1", rec.Code, rec.Body) + } + rec = doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, recordingWebM, nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"duplicate":true`) { + t.Fatalf("idempotent resend = %d %s", rec.Code, rec.Body) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, append(append([]byte{}, recordingWebM...), 'x'), nil); rec.Code != http.StatusConflict { + t.Fatalf("resend with other bytes = %d, want 409", rec.Code) + } + tail := []byte("cluster-two") + if rec := doRecording(s, http.MethodPost, chunkPath(id, 1), config.RoleReadWrite, tail, nil); rec.Code != http.StatusOK { + t.Fatalf("chunk 1 = %d %s", rec.Code, rec.Body) + } + if rec := doRecording(s, http.MethodGet, "/api/inbox/recordings/"+id+"/audio", config.RoleReadWrite, nil, nil); rec.Code != http.StatusNotFound { + t.Fatalf("audio before finalize = %d, want 404", rec.Code) + } + rec = doRecording(s, http.MethodPost, "/api/inbox/recordings/"+id+"/finalize", config.RoleReadWrite, []byte(`{"duration_ms":1500}`), nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"state":"finalized"`) { + t.Fatalf("finalize = %d %s", rec.Code, rec.Body) + } + if e := s.audit.Recent(1); len(e) != 1 || e[0].Action != "inbox_recording_finalize" { + t.Fatalf("finalize not audited: %+v", e) + } + whole := append(append([]byte{}, recordingWebM...), tail...) + + audioPath := "/api/inbox/recordings/" + id + "/audio" + rec = doRecording(s, http.MethodGet, audioPath, config.RoleReadWrite, nil, nil) + if rec.Code != http.StatusOK || !bytes.Equal(rec.Body.Bytes(), whole) { + t.Fatalf("audio = %d %q", rec.Code, rec.Body.Bytes()) + } + for k, want := range map[string]string{"Cache-Control": "no-store", "Content-Type": "audio/webm", "X-Content-Type-Options": "nosniff", "Accept-Ranges": "bytes", "Cross-Origin-Resource-Policy": "same-origin"} { + if got := rec.Header().Get(k); got != want { + t.Fatalf("%s = %q, want %q", k, got, want) + } + } + if e := s.audit.Recent(1); len(e) != 1 || e[0].Action != "inbox_recording_audio" { + t.Fatalf("playback not audited: %+v", e) + } + audited := len(s.audit.Recent(0)) + rec = doRecording(s, http.MethodGet, audioPath, config.RoleReadWrite, nil, map[string]string{"Range": "bytes=4-9"}) + if rec.Code != http.StatusPartialContent || !bytes.Equal(rec.Body.Bytes(), whole[4:10]) || rec.Header().Get("Content-Range") != "bytes 4-9/"+strconv.Itoa(len(whole)) { + t.Fatalf("range = %d %q %q", rec.Code, rec.Body.Bytes(), rec.Header().Get("Content-Range")) + } + if rec.Header().Get("Cache-Control") != "no-store" { + t.Fatal("range response is cacheable") + } + if n := len(s.audit.Recent(0)); n != audited { + t.Fatalf("a seek was audited again: %d entries, want %d", n, audited) + } + + // The Inbox player's generic stream path serves the same file from disk. + stream := doRecording(s, http.MethodGet, "/api/inbox/"+inbox.RecordingType+"/"+id+"/audio/stream", config.RoleReadWrite, nil, map[string]string{"Range": "bytes=0-3"}) + if stream.Code != http.StatusPartialContent || !bytes.Equal(stream.Body.Bytes(), whole[:4]) || stream.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("stream = %d %q", stream.Code, stream.Body.Bytes()) + } + list := doRecording(s, http.MethodGet, "/api/inbox", config.RoleReadWrite, nil, nil) + if !strings.Contains(list.Body.String(), `"source":"`+inbox.RecordingType+`"`) || !strings.Contains(list.Body.String(), `"recorded_by":"read-write@example.com"`) || !strings.Contains(list.Body.String(), `"recording":{"enabled":true`) { + t.Fatalf("inbox listing lacks the recording: %s", list.Body) + } + + // Retention: after the audio is deleted the endpoint answers 410. + src.Store.Now = func() time.Time { return time.Now().Add(config.DefaultInboxRecordingRetentionDays * 24 * time.Hour) } + if res, err := svc.RecordingSweep(inbox.RecordingOptions{}); err != nil || len(res.Expired) != 1 { + t.Fatalf("sweep = %+v, %v", res, err) + } + rec = doRecording(s, http.MethodGet, audioPath, config.RoleReadWrite, nil, nil) + if rec.Code != http.StatusGone || !strings.Contains(rec.Body.String(), "audio expired (kept 5 days)") || rec.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("expired audio = %d %s", rec.Code, rec.Body) + } + if rec := doRecording(s, http.MethodGet, "/api/inbox/"+inbox.RecordingType+"/"+id+"/audio/stream", config.RoleReadWrite, nil, nil); rec.Code != http.StatusGone { + t.Fatalf("expired stream = %d, want 410", rec.Code) + } +} + +func TestInboxRecordingAbortAndCaps(t *testing.T) { + s, _, _ := newRecordingTestServer(t, config.InboxRecording{MaxBytes: 30, QuotaBytes: 40}) + id := startTestRecording(t, s, config.RoleReadWrite) + if rec := doRecording(s, http.MethodPost, chunkPath(id, 0), config.RoleReadWrite, recordingWebM, nil); rec.Code != http.StatusOK { + t.Fatalf("chunk 0 = %d %s", rec.Code, rec.Body) + } + if rec := doRecording(s, http.MethodPost, chunkPath(id, 1), config.RoleReadWrite, []byte("0123456789"), nil); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("over max_bytes = %d, want 413", rec.Code) + } + other := startTestRecording(t, s, config.RoleOwner) + if rec := doRecording(s, http.MethodPost, chunkPath(other, 0), config.RoleOwner, recordingWebM, nil); rec.Code != http.StatusInsufficientStorage { + t.Fatalf("over quota = %d %s, want 507", rec.Code, rec.Body) + } + if rec := doRecording(s, http.MethodDelete, "/api/inbox/recordings/"+id, config.RoleOwner, nil, nil); rec.Code != http.StatusOK { + t.Fatalf("owner abort = %d %s", rec.Code, rec.Body) + } + if e := s.audit.Recent(1); len(e) != 1 || e[0].Action != "inbox_recording_abort" { + t.Fatalf("abort not audited: %+v", e) + } + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings/"+id+"/finalize", config.RoleReadWrite, nil, nil); rec.Code != http.StatusNotFound { + t.Fatalf("finalize aborted = %d, want 404", rec.Code) + } + if rec := doRecording(s, http.MethodPost, "/api/inbox/recordings/"+other+"/finalize", config.RoleOwner, nil, nil); rec.Code != http.StatusBadRequest { + t.Fatalf("finalize empty = %d, want 400", rec.Code) + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index bdb4da9418..987e243102 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -622,3 +622,9 @@ inbox.rules[].title_prefix # see inbox.rules[].source inbox.rules[].tag # see inbox.rules[].source inbox.rules[].destination # see inbox.rules[].source inbox.rules[].repo # see inbox.rules[].source +inbox.recording.enabled # ● Record (dashboard recordings) limits are yaml-only; the recorder reads them from GET /api/inbox settings.recording (see docs/inbox-recording.md) +inbox.recording.retention_days # see inbox.recording.enabled +inbox.recording.max_seconds # see inbox.recording.enabled +inbox.recording.max_bytes # see inbox.recording.enabled +inbox.recording.quota_bytes # see inbox.recording.enabled +inbox.recording.idle_finalize_minutes # see inbox.recording.enabled diff --git a/src/pkg/inbox/factwriteback.go b/src/pkg/inbox/factwriteback.go index e1498e4426..c1cb55b310 100644 --- a/src/pkg/inbox/factwriteback.go +++ b/src/pkg/inbox/factwriteback.go @@ -33,6 +33,7 @@ const ( var factPendingNotes = []string{ "_" + connector.PlaudPendingText + "_", "_" + connector.MeetingPendingText + "_", + "_" + RecordingPendingText + "_", } // factState is the Item.FactState of a fact: its recorded content state, or diff --git a/src/pkg/inbox/inbox.go b/src/pkg/inbox/inbox.go index 907e58eaf4..098eda89fc 100644 --- a/src/pkg/inbox/inbox.go +++ b/src/pkg/inbox/inbox.go @@ -78,6 +78,13 @@ type Item struct { AddedBy string `json:"added_by,omitempty"` // UploadKind is the detected type of an upload (markdown, json, ...). UploadKind string `json:"upload_kind,omitempty"` + // RecordedBy is the dashboard user who made a hive recording. + RecordedBy string `json:"recorded_by,omitempty"` + // AudioExpired reports that retention deleted the raw audio of a hive + // recording (the transcript and fact stay); there is nothing to play. + AudioExpired bool `json:"audio_expired,omitempty"` + // AudioRetentionDays is how long a hive recording's audio is kept. + AudioRetentionDays int `json:"audio_retention_days,omitempty"` } // Key returns the stable "source/id" key used by the state store. diff --git a/src/pkg/inbox/recordings.go b/src/pkg/inbox/recordings.go new file mode 100644 index 0000000000..6319998d17 --- /dev/null +++ b/src/pkg/inbox/recordings.go @@ -0,0 +1,785 @@ +package inbox + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// Recordings made in the dashboard (● Record). The browser uploads the +// MediaRecorder output in small, ordered chunks while it records; each chunk +// is fsynced as its own part file, and finalize concatenates the parts into +// one audio file. The hive never decodes the audio itself: it only checks +// the container signature on the first chunk, and the transcription +// provider (the sandboxed whisper subprocess by default) does the decoding. +const ( + // RecordingType is the Inbox source type of dashboard recordings. + RecordingType = "hive-recording" + // RecordingConnector is the connector name written into their facts. + RecordingConnector = "hive" + // RecordingConsentVersion names the consent notice the dashboard shows + // before every recording. A start request must acknowledge this version; + // it is stored in the recording's metadata with the user. + RecordingConsentVersion = "2026-11-v1" + // RecordingMaxChunkBytes caps one uploaded chunk (512 KiB), well under + // the 1 MiB ingress body default and quick enough for the dashboard's + // 30 s read timeout on a slow uplink. + RecordingMaxChunkBytes = 512 << 10 + // RecordingChunkMillis is the MediaRecorder timeslice the dashboard + // uses: one chunk every 10 s (about 40 KB at 32 kbit/s). + RecordingChunkMillis = 10000 + // RecordingAudioBitsPerSecond is the bitrate the dashboard asks for. + RecordingAudioBitsPerSecond = 32000 + // RecordingTitleRunes bounds a recording title. + RecordingTitleRunes = 200 + // RecordingPendingText is the note a recording's fact carries until it + // has a transcript. + RecordingPendingText = "Recorded in this hive; no transcript yet." + + // Recording states. + RecordingActive = "recording" + RecordingFinalized = "finalized" + + // Accepted containers (sniffed from the first chunk). + RecordingContainerWebM = "webm" + RecordingContainerMP4 = "mp4" + + recordingDirMode = 0o700 + recordingFileMode = 0o600 + recordingMetaFile = "meta.json" + // recordingClockGrace tolerates clock and upload lag past max_seconds. + recordingClockGrace = 5 * time.Minute + // recordingIDRandBytes is the random suffix length of a recording id. + recordingIDRandBytes = 4 + // recordingIDTimeLayout is the sortable time prefix of a recording id. + recordingIDTimeLayout = "20060102t150405" + // mp4BoxHeader is the size+type header of an ISO BMFF box; a size of 1 + // means a 64-bit size follows (mp4LargeBoxHeader in all). + mp4BoxHeader = 8 + mp4LargeBoxHeader = 16 +) + +// recordingIDRe matches the ids NewRecording mints, so an id is always a +// safe single path element. +var recordingIDRe = regexp.MustCompile(`^[0-9]{8}t[0-9]{6}-[0-9a-f]{8}$`) + +// webmMagic is the EBML header every WebM/Matroska file starts with. +var webmMagic = []byte{0x1A, 0x45, 0xDF, 0xA3} + +// recordingMIMEs maps the accepted MediaRecorder base types to containers. +var recordingMIMEs = map[string]string{ + "audio/webm": RecordingContainerWebM, + "audio/mp4": RecordingContainerMP4, +} + +// Recording errors. The dashboard maps each to an HTTP status. +var ( + ErrRecordingDisabled = errors.New("recording is not enabled on this hive (inbox.recording.enabled)") + ErrRecordingNotFound = errors.New("unknown recording") + ErrRecordingNotYours = errors.New("this recording was started by someone else") + ErrRecordingClosed = errors.New("this recording has already ended") + ErrRecordingConsent = errors.New("the recording consent notice must be acknowledged (consent_version " + RecordingConsentVersion + ")") + ErrRecordingMIME = errors.New("unsupported recording type: only audio/webm and audio/mp4 are accepted") + ErrRecordingNotMedia = errors.New("the first chunk is not a WebM or MP4 recording") + ErrRecordingTooLarge = errors.New("the recording is larger than inbox.recording.max_bytes") + ErrRecordingTooLong = errors.New("the recording is longer than inbox.recording.max_seconds") + ErrRecordingQuota = errors.New("this hive's recording storage is full (inbox.recording.quota_bytes)") + ErrRecordingChunkSize = errors.New("a chunk must be between 1 byte and the max_chunk_bytes the start response returned") + ErrRecordingMismatch = errors.New("a chunk with this seq was already stored with different bytes") + ErrRecordingEmpty = errors.New("no audio was received for this recording") + ErrRecordingExpired = errors.New("the audio of this recording has expired") + ErrRecordingBadRequest = errors.New("invalid recording request") +) + +// RecordingActiveError is returned when the user already has a recording in +// progress (one at a time per user); ID lets the dashboard resume it. +type RecordingActiveError struct{ ID string } + +func (e *RecordingActiveError) Error() string { + return "you already have a recording in progress (" + e.ID + "); stop it before starting another" +} + +// RecordingGapError is an out-of-order chunk: Expected is the next seq the +// hive will accept. +type RecordingGapError struct { + Got, Expected int +} + +func (e *RecordingGapError) Error() string { + return fmt.Sprintf("chunk %d is out of order: the next chunk must be seq=%d", e.Got, e.Expected) +} + +// RecordingMeta is one recording's metadata (meta.json). It never holds the +// audio itself. +type RecordingMeta struct { + ID string `json:"id"` + Title string `json:"title"` + MIME string `json:"mime"` + Container string `json:"container,omitempty"` + RecordedBy string `json:"recorded_by"` + Role string `json:"role,omitempty"` + // ConsentVersion / ConsentAt record which notice the recorder + // acknowledged before this recording, and when. + ConsentVersion string `json:"consent_version"` + ConsentAt time.Time `json:"consent_at"` + IncludesTabAudio bool `json:"includes_tab_audio"` + State string `json:"state"` + CreatedAt time.Time `json:"created_at"` + LastChunkAt time.Time `json:"last_chunk_at,omitzero"` + FinalizedAt time.Time `json:"finalized_at,omitzero"` + AutoFinalized bool `json:"auto_finalized,omitempty"` + Chunks int `json:"chunks"` + Bytes int64 `json:"bytes"` + // ChunkSHA256 holds each stored chunk's digest, by seq, so a re-sent + // chunk is recognised. + ChunkSHA256 []string `json:"chunk_sha256,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + AudioFile string `json:"audio_file,omitempty"` + // AudioDeletedAt is when retention removed the audio file. + AudioDeletedAt time.Time `json:"audio_deleted_at,omitzero"` + // ConcatWarning is set when the concatenated file's container structure + // did not check out (the file is kept; the provider decides). + ConcatWarning string `json:"concat_warning,omitempty"` +} + +// AudioExpired reports that retention deleted the audio. +func (m RecordingMeta) AudioExpired() bool { return !m.AudioDeletedAt.IsZero() } + +// audioBytes is what the recording occupies against the quota. +func (m RecordingMeta) audioBytes() int64 { + if m.AudioExpired() { + return 0 + } + return m.Bytes +} + +// RecordingStart is a start request. +type RecordingStart struct { + Title string + MIME string + ConsentVersion string + IncludesTabAudio bool + User string + Role string +} + +// RecordingChunk is a stored (or recognised) chunk. +type RecordingChunk struct { + Seq int `json:"seq"` + NextSeq int `json:"next_seq"` + Bytes int64 `json:"bytes"` + Duplicate bool `json:"duplicate,omitempty"` +} + +// RecordingStore keeps recordings under Dir// (0700 directories, 0600 +// files): part-.bin while recording, then audio.webm or audio.mp4 plus +// meta.json. It is safe for concurrent use. +type RecordingStore struct { + Dir string + // Now is injectable for tests. + Now func() time.Time + + mu sync.Mutex + loaded bool + metas map[string]*RecordingMeta +} + +// NewRecordingStore returns a store rooted at dir. +func NewRecordingStore(dir string) *RecordingStore { return &RecordingStore{Dir: dir} } + +func (s *RecordingStore) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +// load reads every meta.json once. Callers hold s.mu. +func (s *RecordingStore) load() { + if s.loaded { + return + } + s.loaded = true + s.metas = map[string]*RecordingMeta{} + entries, err := os.ReadDir(s.Dir) + if err != nil { + return + } + for _, e := range entries { + if !e.IsDir() || !recordingIDRe.MatchString(e.Name()) { + continue + } + data, err := os.ReadFile(filepath.Join(s.Dir, e.Name(), recordingMetaFile)) + if err != nil { + continue + } + var m RecordingMeta + if json.Unmarshal(data, &m) != nil || m.ID != e.Name() { + continue + } + s.metas[m.ID] = &m + } +} + +func (s *RecordingStore) dir(id string) string { return filepath.Join(s.Dir, id) } + +func partName(seq int) string { return fmt.Sprintf("part-%06d.bin", seq) } + +// usedBytes sums the audio held on disk. Callers hold s.mu. +func (s *RecordingStore) usedBytes() int64 { + var n int64 + for _, m := range s.metas { + n += m.audioBytes() + } + return n +} + +// UsedBytes returns the recording audio currently kept on this hive. +func (s *RecordingStore) UsedBytes() int64 { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + return s.usedBytes() +} + +// RecordingContainer returns the container for a MediaRecorder MIME type +// ("audio/webm;codecs=opus" → webm), or "" when it is not accepted. +func RecordingContainer(mimeType string) string { + base, _, err := mime.ParseMediaType(strings.TrimSpace(mimeType)) + if err != nil { + return "" + } + return recordingMIMEs[strings.ToLower(base)] +} + +// SniffRecordingContainer names the container a chunk starts with: webm for +// the EBML header, mp4 for an ISO BMFF ftyp box, "" for anything else. +func SniffRecordingContainer(head []byte) string { + switch { + case bytes.HasPrefix(head, webmMagic): + return RecordingContainerWebM + case len(head) >= mp4BoxHeader && string(head[4:8]) == "ftyp": + return RecordingContainerMP4 + } + return "" +} + +// newRecordingID mints a sortable, path-safe id. +func newRecordingID(now time.Time) (string, error) { + b := make([]byte, recordingIDRandBytes) + if _, err := rand.Read(b); err != nil { + return "", err + } + return now.UTC().Format(recordingIDTimeLayout) + "-" + hex.EncodeToString(b), nil +} + +// Create starts a recording for req.User. Only one recording per user may +// be in progress; consent must name RecordingConsentVersion. +func (s *RecordingStore) Create(req RecordingStart, cfg config.InboxRecording) (RecordingMeta, error) { + if strings.TrimSpace(req.ConsentVersion) != RecordingConsentVersion { + return RecordingMeta{}, ErrRecordingConsent + } + container := RecordingContainer(req.MIME) + if container == "" { + return RecordingMeta{}, ErrRecordingMIME + } + if strings.TrimSpace(req.User) == "" { + return RecordingMeta{}, ErrRecordingBadRequest + } + s.mu.Lock() + defer s.mu.Unlock() + s.load() + for _, m := range s.metas { + if m.State == RecordingActive && m.RecordedBy == req.User { + return RecordingMeta{}, &RecordingActiveError{ID: m.ID} + } + } + if s.usedBytes() >= cfg.EffectiveQuotaBytes() { + return RecordingMeta{}, ErrRecordingQuota + } + now := s.now() + id, err := newRecordingID(now) + if err != nil { + return RecordingMeta{}, err + } + title := truncateRunes(strings.TrimSpace(req.Title), RecordingTitleRunes) + if title == "" { + title = "Recording " + now.Format("2006-01-02 15:04") + " UTC" + } + m := &RecordingMeta{ + ID: id, Title: title, MIME: strings.TrimSpace(req.MIME), Container: container, + RecordedBy: req.User, Role: req.Role, ConsentVersion: RecordingConsentVersion, ConsentAt: now, + IncludesTabAudio: req.IncludesTabAudio, State: RecordingActive, CreatedAt: now, + } + if err := os.MkdirAll(s.Dir, recordingDirMode); err != nil { + return RecordingMeta{}, err + } + if err := os.Mkdir(s.dir(id), recordingDirMode); err != nil { + return RecordingMeta{}, err + } + if err := s.writeMeta(m); err != nil { + _ = os.RemoveAll(s.dir(id)) + return RecordingMeta{}, err + } + s.metas[id] = m + return *m, nil +} + +// get returns the meta for id owned by user. Callers hold s.mu. +func (s *RecordingStore) get(id, user string) (*RecordingMeta, error) { + s.load() + m, ok := s.metas[id] + if !ok { + return nil, ErrRecordingNotFound + } + if user != "" && m.RecordedBy != user { + return nil, ErrRecordingNotYours + } + return m, nil +} + +// Append stores chunk seq of recording id. Chunks are accepted in order +// only: a gap is a *RecordingGapError naming the expected seq, and a re-sent +// chunk with the same bytes (same sha256) is acknowledged again without +// being written twice. Each part is fsynced before the ack. +func (s *RecordingStore) Append(id, user string, seq int, data []byte, cfg config.InboxRecording) (RecordingChunk, error) { + if len(data) == 0 || len(data) > RecordingMaxChunkBytes { + return RecordingChunk{}, ErrRecordingChunkSize + } + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.get(id, user) + if err != nil { + return RecordingChunk{}, err + } + sum := sha256.Sum256(data) + digest := hex.EncodeToString(sum[:]) + if seq >= 0 && seq < m.Chunks { + if seq < len(m.ChunkSHA256) && m.ChunkSHA256[seq] == digest { + return RecordingChunk{Seq: seq, NextSeq: m.Chunks, Bytes: m.Bytes, Duplicate: true}, nil + } + return RecordingChunk{}, ErrRecordingMismatch + } + if m.State != RecordingActive { + return RecordingChunk{}, ErrRecordingClosed + } + if seq != m.Chunks { + return RecordingChunk{}, &RecordingGapError{Got: seq, Expected: m.Chunks} + } + if seq == 0 && SniffRecordingContainer(data) != m.Container { + return RecordingChunk{}, ErrRecordingNotMedia + } + now := s.now() + if now.Sub(m.CreatedAt) > time.Duration(cfg.EffectiveMaxSeconds())*time.Second+recordingClockGrace { + return RecordingChunk{}, ErrRecordingTooLong + } + if m.Bytes+int64(len(data)) > cfg.EffectiveMaxBytes() { + return RecordingChunk{}, ErrRecordingTooLarge + } + if s.usedBytes()+int64(len(data)) > cfg.EffectiveQuotaBytes() { + return RecordingChunk{}, ErrRecordingQuota + } + if err := writeFileSync(filepath.Join(s.dir(id), partName(seq)), data); err != nil { + return RecordingChunk{}, err + } + next := *m + next.Chunks++ + next.Bytes += int64(len(data)) + next.ChunkSHA256 = append(append([]string(nil), m.ChunkSHA256...), digest) + next.LastChunkAt = now + if err := s.writeMeta(&next); err != nil { + _ = os.Remove(filepath.Join(s.dir(id), partName(seq))) + return RecordingChunk{}, err + } + *m = next + return RecordingChunk{Seq: seq, NextSeq: m.Chunks, Bytes: m.Bytes}, nil +} + +// Finalize ends recording id: the parts are concatenated in order into +// audio. and removed. durationMS is the recorder's own count +// (pauses excluded); it is clamped to the recording's wall-clock span and +// max_seconds, and replaced by that span when the client sent none. user "" +// is the idle sweep. A recording with no chunk is removed and reported as +// ErrRecordingEmpty. +func (s *RecordingStore) Finalize(id, user string, durationMS int64, cfg config.InboxRecording) (RecordingMeta, error) { + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.get(id, user) + if err != nil { + return RecordingMeta{}, err + } + if m.State != RecordingActive { + return RecordingMeta{}, ErrRecordingClosed + } + if m.Chunks == 0 { + _ = os.RemoveAll(s.dir(id)) + delete(s.metas, id) + return RecordingMeta{}, ErrRecordingEmpty + } + next := *m + next.AudioFile = "audio." + m.Container + warn, err := s.concat(&next) + if err != nil { + return RecordingMeta{}, err + } + next.ConcatWarning = warn + span := next.LastChunkAt.Sub(next.CreatedAt).Milliseconds() + maxMS := int64(cfg.EffectiveMaxSeconds()) * int64(time.Second/time.Millisecond) + if durationMS <= 0 || (span > 0 && durationMS > span+int64(RecordingChunkMillis)) { + durationMS = span + } + if durationMS > maxMS { + durationMS = maxMS + } + if durationMS < 0 { + durationMS = 0 + } + next.DurationMS = durationMS + next.State = RecordingFinalized + next.FinalizedAt = s.now() + next.AutoFinalized = user == "" + if err := s.writeMeta(&next); err != nil { + return RecordingMeta{}, err + } + for seq := 0; seq < next.Chunks; seq++ { + _ = os.Remove(filepath.Join(s.dir(id), partName(seq))) + } + *m = next + return *m, nil +} + +// concat writes the parts of m into its audio file through a temp file, +// fsyncs and renames it. It returns a warning when the container structure +// of the result does not check out. +func (s *RecordingStore) concat(m *RecordingMeta) (string, error) { + dir := s.dir(m.ID) + tmp, err := os.CreateTemp(dir, ".audio-*") + if err != nil { + return "", err + } + tmpName := tmp.Name() + fail := func(err error) (string, error) { + _ = tmp.Close() + _ = os.Remove(tmpName) + return "", err + } + if err := tmp.Chmod(recordingFileMode); err != nil { + return fail(err) + } + for seq := 0; seq < m.Chunks; seq++ { + part, err := os.Open(filepath.Join(dir, partName(seq))) + if err != nil { + return fail(fmt.Errorf("chunk %d: %w", seq, err)) + } + _, err = io.Copy(tmp, part) + _ = part.Close() + if err != nil { + return fail(err) + } + } + if err := tmp.Sync(); err != nil { + return fail(err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return "", err + } + final := filepath.Join(dir, m.AudioFile) + if err := os.Rename(tmpName, final); err != nil { + _ = os.Remove(tmpName) + return "", err + } + syncDir(dir) + return checkRecordingContainer(final, m.Container), nil +} + +// checkRecordingContainer verifies the concatenated file: for mp4 every +// top-level ISO BMFF box must chain exactly to the end of the file, starting +// with ftyp (fragmented MP4 from Safari is ftyp, moov, then moof+mdat pairs); +// for webm the EBML header must lead. It never decodes audio. +func checkRecordingContainer(path, container string) string { + f, err := os.Open(path) + if err != nil { + return err.Error() + } + defer f.Close() + if container == RecordingContainerWebM { + head := make([]byte, len(webmMagic)) + if _, err := io.ReadFull(f, head); err != nil || !bytes.Equal(head, webmMagic) { + return "the recording does not start with a WebM header" + } + return "" + } + types, err := MP4TopLevelBoxes(f) + if err != nil { + return err.Error() + } + if len(types) == 0 || types[0] != "ftyp" { + return "the recording does not start with an MP4 ftyp box" + } + return "" +} + +// MP4TopLevelBoxes walks the top-level boxes of an ISO BMFF stream and +// returns their types in order. It fails when a box size overruns the +// stream or is malformed, which is how a broken concatenation shows. +func MP4TopLevelBoxes(r io.ReadSeeker) ([]string, error) { + var types []string + end, err := r.Seek(0, io.SeekEnd) + if err != nil { + return nil, err + } + var off int64 + hdr := make([]byte, mp4LargeBoxHeader) + for off < end { + if end-off < mp4BoxHeader { + return types, fmt.Errorf("mp4: %d trailing bytes after the last box", end-off) + } + if _, err := r.Seek(off, io.SeekStart); err != nil { + return types, err + } + if _, err := io.ReadFull(r, hdr[:mp4BoxHeader]); err != nil { + return types, err + } + size := int64(binary.BigEndian.Uint32(hdr[:4])) + typ := string(hdr[4:8]) + switch size { + case 0: + size = end - off + case 1: + if _, err := io.ReadFull(r, hdr[mp4BoxHeader:mp4LargeBoxHeader]); err != nil { + return types, err + } + size = int64(binary.BigEndian.Uint64(hdr[mp4BoxHeader:mp4LargeBoxHeader])) + if size < mp4LargeBoxHeader { + return types, fmt.Errorf("mp4: box %q has an invalid size", typ) + } + default: + if size < mp4BoxHeader { + return types, fmt.Errorf("mp4: box %q has an invalid size", typ) + } + } + if off+size > end { + return types, fmt.Errorf("mp4: box %q overruns the file", typ) + } + types = append(types, typ) + off += size + } + return types, nil +} + +// Abort removes an in-progress recording. The recorder may abort their own; +// an owner (asOwner) may abort anyone's. +func (s *RecordingStore) Abort(id, user string, asOwner bool) (RecordingMeta, error) { + s.mu.Lock() + defer s.mu.Unlock() + owner := user + if asOwner { + owner = "" + } + m, err := s.get(id, owner) + if err != nil { + return RecordingMeta{}, err + } + if m.State != RecordingActive { + return RecordingMeta{}, ErrRecordingClosed + } + out := *m + if err := os.RemoveAll(s.dir(id)); err != nil { + return RecordingMeta{}, err + } + delete(s.metas, id) + return out, nil +} + +// Get returns one recording's metadata. +func (s *RecordingStore) Get(id string) (RecordingMeta, bool) { + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.get(id, "") + if err != nil { + return RecordingMeta{}, false + } + return *m, true +} + +// List returns every recording, newest first. +func (s *RecordingStore) List() []RecordingMeta { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + out := make([]RecordingMeta, 0, len(s.metas)) + for _, m := range s.metas { + out = append(out, *m) + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) }) + return out +} + +// OpenAudio opens a finalized recording's audio file for reading. +func (s *RecordingStore) OpenAudio(id string) (*os.File, RecordingMeta, error) { + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.get(id, "") + if err != nil { + return nil, RecordingMeta{}, err + } + if m.State != RecordingFinalized || m.AudioFile == "" { + return nil, *m, ErrNoAudio + } + if m.AudioExpired() { + return nil, *m, ErrRecordingExpired + } + f, err := os.Open(filepath.Join(s.dir(id), m.AudioFile)) + if err != nil { + return nil, *m, err + } + return f, *m, nil +} + +// RecordingSweepResult is what one sweep changed. +type RecordingSweepResult struct { + // Finalized were in progress and idle past idle_finalize_minutes. + Finalized []RecordingMeta + // Removed were idle with no audio at all. + Removed []RecordingMeta + // Expired had their audio deleted by retention. + Expired []RecordingMeta +} + +// Sweep finalizes idle recordings and deletes audio past retention. The +// metadata, the transcript and the Knowledge fact are kept. +func (s *RecordingStore) Sweep(cfg config.InboxRecording) (RecordingSweepResult, error) { + var res RecordingSweepResult + var errs []error + now := s.now() + for _, m := range s.List() { + switch { + case m.State == RecordingActive: + last := m.LastChunkAt + if last.IsZero() { + last = m.CreatedAt + } + if now.Sub(last) < cfg.EffectiveIdleFinalize() { + continue + } + fin, err := s.Finalize(m.ID, "", 0, cfg) + switch { + case errors.Is(err, ErrRecordingEmpty): + res.Removed = append(res.Removed, m) + case err != nil: + errs = append(errs, fmt.Errorf("finalize %s: %w", m.ID, err)) + default: + res.Finalized = append(res.Finalized, fin) + } + case m.State == RecordingFinalized && !m.AudioExpired() && m.AudioFile != "": + if now.Sub(m.FinalizedAt) < cfg.EffectiveRetention() { + continue + } + exp, err := s.expire(m.ID) + if err != nil { + errs = append(errs, fmt.Errorf("expire %s: %w", m.ID, err)) + continue + } + res.Expired = append(res.Expired, exp) + } + } + return res, errors.Join(errs...) +} + +// expire deletes one recording's audio file and records when. +func (s *RecordingStore) expire(id string) (RecordingMeta, error) { + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.get(id, "") + if err != nil { + return RecordingMeta{}, err + } + if err := os.Remove(filepath.Join(s.dir(id), m.AudioFile)); err != nil && !os.IsNotExist(err) { + return RecordingMeta{}, err + } + next := *m + next.AudioDeletedAt = s.now() + if err := s.writeMeta(&next); err != nil { + return RecordingMeta{}, err + } + *m = next + return *m, nil +} + +// writeMeta writes meta.json through a temp file, fsync and rename. +func (s *RecordingStore) writeMeta(m *RecordingMeta) error { + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + dir := s.dir(m.ID) + tmp, err := os.CreateTemp(dir, ".meta-*") + if err != nil { + return err + } + name := tmp.Name() + _, werr := tmp.Write(data) + if werr == nil { + werr = tmp.Chmod(recordingFileMode) + } + if werr == nil { + werr = tmp.Sync() + } + if cerr := tmp.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Rename(name, filepath.Join(dir, recordingMetaFile)) + } + if werr != nil { + _ = os.Remove(name) + return werr + } + syncDir(dir) + return nil +} + +// writeFileSync writes data to path (0600) and fsyncs it. +func writeFileSync(path string, data []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, recordingFileMode) + if err != nil { + return err + } + _, werr := f.Write(data) + if werr == nil { + werr = f.Sync() + } + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr != nil { + _ = os.Remove(path) + } + return werr +} + +// syncDir fsyncs a directory so a rename or new file survives a crash. It +// is best effort: some filesystems refuse directory fsync. +func syncDir(dir string) { + if d, err := os.Open(dir); err == nil { + _ = d.Sync() + _ = d.Close() + } +} diff --git a/src/pkg/inbox/recordings_test.go b/src/pkg/inbox/recordings_test.go new file mode 100644 index 0000000000..2bf68c3a1b --- /dev/null +++ b/src/pkg/inbox/recordings_test.go @@ -0,0 +1,527 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// recordingClock is a settable test clock. +type recordingClock struct{ t time.Time } + +func (c *recordingClock) now() time.Time { return c.t } + +func enabledRecording() config.InboxRecording { + on := true + return config.InboxRecording{Enabled: &on} +} + +// webmChunk returns a chunk that starts like a MediaRecorder WebM stream. +func webmChunk(n int) []byte { + b := append([]byte{}, webmMagic...) + return append(b, bytes.Repeat([]byte{0x42}, n)...) +} + +// mp4Box builds one ISO BMFF box. +func mp4Box(typ string, payload []byte) []byte { + b := make([]byte, mp4BoxHeader, mp4BoxHeader+len(payload)) + binary.BigEndian.PutUint32(b[:4], uint32(mp4BoxHeader+len(payload))) + copy(b[4:8], typ) + return append(b, payload...) +} + +func newRecordingService(t *testing.T, cfg config.InboxRecording) (*Service, *RecordingSource, *recordingClock) { + t.Helper() + clock := &recordingClock{t: time.Date(2026, 11, 16, 9, 0, 0, 0, time.UTC)} + dir := t.TempDir() + store := NewRecordingStore(filepath.Join(dir, "recordings")) + store.Now = clock.now + src := &RecordingSource{Store: store, FactDir: filepath.Join(dir, "connectors", RecordingLayer)} + svc := NewService(config.InboxConfig{Recording: cfg}, NewStore(filepath.Join(dir, "_inbox.json")), nil, nil, nil) + svc.Now = clock.now + svc.Run = func(fn func()) { fn() } + svc.EnableTranscription(NewTranscriptStore(filepath.Join(dir, "transcripts"))) + svc.EnableRecordings(src) + return svc, src, clock +} + +func startRecording(t *testing.T, svc *Service, user, mime string) RecordingMeta { + t.Helper() + m, err := svc.StartRecording(RecordingStart{Title: "Standup", MIME: mime, ConsentVersion: RecordingConsentVersion, User: user, Role: "read-write"}, RecordingOptions{}) + if err != nil { + t.Fatalf("start: %v", err) + } + return m +} + +func TestRecordingDisabledByDefault(t *testing.T) { + svc, _, _ := newRecordingService(t, config.InboxRecording{}) + if _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: RecordingConsentVersion, User: "a"}, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("start with the default flag = %v, want ErrRecordingDisabled", err) + } + if svc.RecordingSettings().Enabled { + t.Fatal("settings report recording enabled with the default flag") + } +} + +func TestRecordingConsentRequiredAndStored(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + for _, v := range []string{"", "old-notice"} { + if _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: v, User: "ana"}, RecordingOptions{}); !errors.Is(err, ErrRecordingConsent) { + t.Fatalf("consent %q = %v, want ErrRecordingConsent", v, err) + } + } + var audits []RecordingAudit + m, err := svc.StartRecording(RecordingStart{Title: "Design review", MIME: "audio/webm;codecs=opus", ConsentVersion: RecordingConsentVersion, IncludesTabAudio: true, User: "ana", Role: "owner"}, + RecordingOptions{Actor: "ana", Audit: func(a RecordingAudit) { audits = append(audits, a) }}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(src.Store.Dir, m.ID, recordingMetaFile)) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"consent_version": "` + RecordingConsentVersion + `"`, `"recorded_by": "ana"`, `"includes_tab_audio": true`, `"consent_at": "` + clock.t.Format(time.RFC3339)} { + if !strings.Contains(string(data), want) { + t.Fatalf("meta.json lacks %s:\n%s", want, data) + } + } + if len(audits) != 1 || audits[0].Action != "inbox_recording_start" || !strings.Contains(audits[0].Detail(), "consent_version="+RecordingConsentVersion) { + t.Fatalf("audits = %+v", audits) + } + if st, err := os.Stat(filepath.Join(src.Store.Dir, m.ID)); err != nil || st.Mode().Perm() != recordingDirMode { + t.Fatalf("recording dir mode = %v, %v", st, err) + } + if st, err := os.Stat(filepath.Join(src.Store.Dir, m.ID, recordingMetaFile)); err != nil || st.Mode().Perm() != recordingFileMode { + t.Fatalf("meta mode = %v, %v", st, err) + } +} + +func TestRecordingOneActivePerUser(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + first := startRecording(t, svc, "ana", "audio/webm") + _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: RecordingConsentVersion, User: "ana"}, RecordingOptions{}) + var active *RecordingActiveError + if !errors.As(err, &active) || active.ID != first.ID { + t.Fatalf("second start = %v, want RecordingActiveError(%s)", err, first.ID) + } + startRecording(t, svc, "ben", "audio/webm") +} + +func TestRecordingMIMEAndContainerSniff(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + for _, mt := range []string{"audio/ogg", "video/webm", "text/plain", "", "audio/wav"} { + if _, err := svc.StartRecording(RecordingStart{MIME: mt, ConsentVersion: RecordingConsentVersion, User: "x"}, RecordingOptions{}); !errors.Is(err, ErrRecordingMIME) { + t.Fatalf("mime %q = %v, want ErrRecordingMIME", mt, err) + } + } + m := startRecording(t, svc, "ana", "audio/webm") + for _, junk := range [][]byte{[]byte("not audio"), []byte("RIFF....WAVEfmt "), mp4Box("ftyp", []byte("isom"))} { + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, junk); !errors.Is(err, ErrRecordingNotMedia) { + t.Fatalf("first chunk %q = %v, want ErrRecordingNotMedia", junk[:4], err) + } + } + mp4 := startRecording(t, svc, "ben", "audio/mp4") + if _, err := svc.AppendRecordingChunk(mp4.ID, "ben", 0, webmChunk(4)); !errors.Is(err, ErrRecordingNotMedia) { + t.Fatalf("webm bytes into an mp4 recording = %v, want ErrRecordingNotMedia", err) + } + if _, err := svc.AppendRecordingChunk(mp4.ID, "ben", 0, mp4Box("ftyp", []byte("iso5"))); err != nil { + t.Fatalf("mp4 first chunk: %v", err) + } +} + +func TestRecordingChunkOrderGapAndIdempotentResend(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ben", 0, webmChunk(8)); !errors.Is(err, ErrRecordingNotYours) { + t.Fatalf("other user's chunk = %v, want ErrRecordingNotYours", err) + } + c0 := webmChunk(8) + res, err := svc.AppendRecordingChunk(m.ID, "ana", 0, c0) + if err != nil || res.NextSeq != 1 || res.Duplicate { + t.Fatalf("chunk 0 = %+v, %v", res, err) + } + _, err = svc.AppendRecordingChunk(m.ID, "ana", 2, []byte("later")) + var gap *RecordingGapError + if !errors.As(err, &gap) || gap.Expected != 1 || gap.Got != 2 { + t.Fatalf("gap = %v, want expected 1", err) + } + res, err = svc.AppendRecordingChunk(m.ID, "ana", 0, c0) + if err != nil || !res.Duplicate || res.NextSeq != 1 { + t.Fatalf("resend = %+v, %v; want duplicate ack", res, err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(9)); !errors.Is(err, ErrRecordingMismatch) { + t.Fatalf("resend with other bytes = %v, want ErrRecordingMismatch", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 1, []byte("second")); err != nil { + t.Fatal(err) + } + got, _ := src.Store.Get(m.ID) + if got.Chunks != 2 || got.Bytes != int64(len(c0)+len("second")) || len(got.ChunkSHA256) != 2 { + t.Fatalf("meta after two chunks = %+v", got) + } + part, err := os.Stat(filepath.Join(src.Store.Dir, m.ID, partName(0))) + if err != nil || part.Mode().Perm() != recordingFileMode { + t.Fatalf("part mode = %v, %v", part, err) + } + // A fresh store (a hive restart) picks the recording up where it was. + again := NewRecordingStore(src.Store.Dir) + again.Now = clock.now + if _, err := again.Append(m.ID, "ana", 2, []byte("third"), enabledRecording()); err != nil { + t.Fatalf("append after reload: %v", err) + } +} + +func TestRecordingSizeCapsAndQuota(t *testing.T) { + cfg := enabledRecording() + cfg.MaxBytes, cfg.QuotaBytes = 40, 60 + svc, _, _ := newRecordingService(t, cfg) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, nil); !errors.Is(err, ErrRecordingChunkSize) { + t.Fatalf("empty chunk = %v", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, make([]byte, RecordingMaxChunkBytes+1)); !errors.Is(err, ErrRecordingChunkSize) { + t.Fatalf("oversize chunk = %v", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(26)); err != nil { + t.Fatal(err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 1, make([]byte, 11)); !errors.Is(err, ErrRecordingTooLarge) { + t.Fatalf("over max_bytes = %v, want ErrRecordingTooLarge", err) + } + other := startRecording(t, svc, "ben", "audio/webm") + if _, err := svc.AppendRecordingChunk(other.ID, "ben", 0, webmChunk(26)); err != nil { + t.Fatal(err) + } + if _, err := svc.AppendRecordingChunk(other.ID, "ben", 1, make([]byte, 6)); !errors.Is(err, ErrRecordingQuota) { + t.Fatalf("over quota_bytes = %v, want ErrRecordingQuota", err) + } + if _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: RecordingConsentVersion, User: "cy"}, RecordingOptions{}); !errors.Is(err, ErrRecordingQuota) { + t.Fatalf("start with the quota used up = %v, want ErrRecordingQuota", err) + } + if svc.RecordingSettings().UsedBytes != 60 { + t.Fatalf("used bytes = %d, want 60", svc.RecordingSettings().UsedBytes) + } +} + +func TestRecordingMaxSeconds(t *testing.T) { + cfg := enabledRecording() + cfg.MaxSeconds = 60 + svc, _, clock := newRecordingService(t, cfg) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(4)); err != nil { + t.Fatal(err) + } + clock.t = clock.t.Add(time.Minute + recordingClockGrace + time.Second) + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 1, []byte("late")); !errors.Is(err, ErrRecordingTooLong) { + t.Fatalf("past max_seconds = %v, want ErrRecordingTooLong", err) + } + fin, err := svc.FinalizeRecording(m.ID, "ana", int64(time.Hour/time.Millisecond), RecordingOptions{}) + if err != nil { + t.Fatal(err) + } + if fin.DurationMS > int64(time.Minute/time.Millisecond) { + t.Fatalf("duration %d not clamped to max_seconds", fin.DurationMS) + } +} + +func TestRecordingFinalizeConcatenatesWebM(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + chunks := [][]byte{webmChunk(5), []byte("cluster-1"), []byte("cluster-2")} + for i, c := range chunks { + clock.t = clock.t.Add(10 * time.Second) + if _, err := svc.AppendRecordingChunk(m.ID, "ana", i, c); err != nil { + t.Fatal(err) + } + } + if _, err := svc.FinalizeRecording(m.ID, "ben", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingNotYours) { + t.Fatalf("finalize by someone else = %v", err) + } + var audits []RecordingAudit + fin, err := svc.FinalizeRecording(m.ID, "ana", 25000, RecordingOptions{Audit: func(a RecordingAudit) { audits = append(audits, a) }}) + if err != nil { + t.Fatal(err) + } + if fin.State != RecordingFinalized || fin.AudioFile != "audio.webm" || fin.DurationMS != 25000 || fin.ConcatWarning != "" { + t.Fatalf("finalized = %+v", fin) + } + got, err := os.ReadFile(filepath.Join(src.Store.Dir, m.ID, "audio.webm")) + if err != nil || !bytes.Equal(got, bytes.Join(chunks, nil)) { + t.Fatalf("audio.webm = %q, %v; want the chunks in order", got, err) + } + if st, _ := os.Stat(filepath.Join(src.Store.Dir, m.ID, "audio.webm")); st.Mode().Perm() != recordingFileMode { + t.Fatalf("audio mode = %v", st.Mode()) + } + if _, err := os.Stat(filepath.Join(src.Store.Dir, m.ID, partName(0))); !os.IsNotExist(err) { + t.Fatalf("parts kept after finalize: %v", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 3, []byte("more")); !errors.Is(err, ErrRecordingClosed) { + t.Fatalf("chunk after finalize = %v, want ErrRecordingClosed", err) + } + if len(audits) != 1 || audits[0].Action != "inbox_recording_finalize" { + t.Fatalf("audits = %+v", audits) + } + // The fact is written pending, in the personal layer. + items, err := svc.Items(context.Background()) + if err != nil || len(items) != 1 { + t.Fatalf("items = %+v, %v", items, err) + } + it := items[0] + if it.Source != RecordingType || it.RecordedBy != "ana" || it.Title != "Standup" || it.DurationMS != 25000 || !it.Pending || it.FactState != connector.FactContentPending || it.Layer != RecordingLayer { + t.Fatalf("item = %+v", it) + } + fact, _ := os.ReadFile(it.FactPath) + for _, want := range []string{"source: " + RecordingType, "attr_hive_recording_content: pending", "attr_consent_version: " + RecordingConsentVersion, "_" + RecordingPendingText + "_"} { + if !strings.Contains(string(fact), want) { + t.Fatalf("fact lacks %q:\n%s", want, fact) + } + } + a, local, err := svc.OpenLocalAudio(context.Background(), it) + if err != nil || !local || a.MIME != "audio/webm" { + t.Fatalf("local audio = %+v, %v, %v", a, local, err) + } + _ = a.File.Close() + link, err := svc.AudioLink(context.Background(), it) + if err != nil || link.URL != "/api/inbox/recordings/"+m.ID+"/audio" { + t.Fatalf("audio link = %+v, %v", link, err) + } +} + +// Safari records fragmented MP4: the first timeslice chunk carries ftyp and +// moov, every later one moof+mdat. Concatenating them in order must give a +// file whose top-level boxes chain exactly to the end. +func TestRecordingFinalizeFragmentedMP4IsValid(t *testing.T) { + svc, src, _ := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/mp4") + chunks := [][]byte{ + append(mp4Box("ftyp", []byte("iso5\x00\x00\x02\x00iso6mp41")), mp4Box("moov", bytes.Repeat([]byte{1}, 40))...), + append(mp4Box("moof", bytes.Repeat([]byte{2}, 24)), mp4Box("mdat", bytes.Repeat([]byte{3}, 300))...), + append(mp4Box("moof", bytes.Repeat([]byte{4}, 24)), mp4Box("mdat", bytes.Repeat([]byte{5}, 200))...), + } + for i, c := range chunks { + if _, err := svc.AppendRecordingChunk(m.ID, "ana", i, c); err != nil { + t.Fatal(err) + } + } + fin, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}) + if err != nil || fin.ConcatWarning != "" || fin.AudioFile != "audio.mp4" { + t.Fatalf("finalize = %+v, %v", fin, err) + } + f, err := os.Open(filepath.Join(src.Store.Dir, m.ID, "audio.mp4")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + types, err := MP4TopLevelBoxes(f) + if err != nil || strings.Join(types, ",") != "ftyp,moov,moof,mdat,moof,mdat" { + t.Fatalf("boxes = %v, %v", types, err) + } + // A truncated chunk breaks the chain and is reported, not hidden. + if _, err := MP4TopLevelBoxes(bytes.NewReader(append(chunks[0], mp4Box("mdat", make([]byte, 50))[:20]...))); err == nil { + t.Fatal("truncated mp4 passed the box walk") + } + if w := checkRecordingContainer(filepath.Join(src.Store.Dir, m.ID, "audio.mp4"), RecordingContainerMP4); w != "" { + t.Fatalf("container check = %q", w) + } +} + +func TestRecordingFinalizeEmptyAndAbort(t *testing.T) { + svc, src, _ := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingEmpty) { + t.Fatalf("finalize without chunks = %v, want ErrRecordingEmpty", err) + } + if _, err := os.Stat(filepath.Join(src.Store.Dir, m.ID)); !os.IsNotExist(err) { + t.Fatal("empty recording kept on disk") + } + m = startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(3)); err != nil { + t.Fatal(err) + } + if _, err := svc.AbortRecording(m.ID, "ben", false, RecordingOptions{}); !errors.Is(err, ErrRecordingNotYours) { + t.Fatalf("abort by another read-write user = %v", err) + } + if _, err := svc.AbortRecording(m.ID, "olivia", true, RecordingOptions{}); err != nil { + t.Fatalf("owner abort: %v", err) + } + if _, ok := src.Store.Get(m.ID); ok { + t.Fatal("aborted recording still listed") + } + if _, err := svc.AbortRecording(m.ID, "ana", false, RecordingOptions{}); !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("abort twice = %v", err) + } +} + +func TestRecordingRetentionSweepKeepsTranscriptAndFact(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(16)); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 1000, RecordingOptions{}); err != nil { + t.Fatal(err) + } + if err := svc.transcriptStore().Put(TranscriptRecord{Source: RecordingType, ID: m.ID, Status: TranscriptionDone, Text: "hello team", Provider: "whisper", Model: "base.en", UpdatedAt: clock.t}); err != nil { + t.Fatal(err) + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].FactState != connector.FactContentMachine { + t.Fatalf("machine transcript not written into the fact: %+v", items) + } + audio := filepath.Join(src.Store.Dir, m.ID, "audio.webm") + + clock.t = clock.t.Add(4*24*time.Hour + 23*time.Hour) + if res, err := svc.RecordingSweep(RecordingOptions{}); err != nil || len(res.Expired) != 0 { + t.Fatalf("sweep before 5 days = %+v, %v", res, err) + } + if _, err := os.Stat(audio); err != nil { + t.Fatalf("audio deleted before retention: %v", err) + } + clock.t = clock.t.Add(time.Hour) + var audits []RecordingAudit + res, err := svc.RecordingSweep(RecordingOptions{Audit: func(a RecordingAudit) { audits = append(audits, a) }}) + if err != nil || len(res.Expired) != 1 { + t.Fatalf("sweep at 5 days = %+v, %v", res, err) + } + if len(audits) != 1 || audits[0].Action != "inbox_recording_audio_expired" || audits[0].Actor != RecordingSweepActor { + t.Fatalf("audits = %+v", audits) + } + if _, err := os.Stat(audio); !os.IsNotExist(err) { + t.Fatalf("audio kept past retention: %v", err) + } + if rec, ok := svc.transcriptStore().Get(RecordingType, m.ID); !ok || rec.Text != "hello team" { + t.Fatal("transcript removed by retention") + } + items, _ = svc.Items(context.Background()) + if len(items) != 1 { + t.Fatalf("item gone after retention: %+v", items) + } + it := items[0] + if !it.AudioExpired || it.AudioRetentionDays != config.DefaultInboxRecordingRetentionDays || it.Transcribable || it.FactState != connector.FactContentMachine { + t.Fatalf("expired item = %+v", it) + } + fact, err := os.ReadFile(it.FactPath) + if err != nil || !strings.Contains(string(fact), "hello team") { + t.Fatalf("fact after retention = %q, %v", fact, err) + } + if _, err := svc.OpenRecordingAudio(m.ID); !errors.Is(err, ErrRecordingExpired) { + t.Fatalf("open expired audio = %v", err) + } + if _, err := svc.AudioLink(context.Background(), it); !errors.Is(err, ErrNoAudio) { + t.Fatalf("audio link after expiry = %v", err) + } + if svc.RecordingSettings().UsedBytes != 0 { + t.Fatal("expired audio still counts against the quota") + } +} + +func TestRecordingRetentionDaysConfigurable(t *testing.T) { + cfg := enabledRecording() + cfg.RetentionDays = 1 + svc, _, clock := newRecordingService(t, cfg) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(4)); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); err != nil { + t.Fatal(err) + } + clock.t = clock.t.Add(24 * time.Hour) + if res, _ := svc.RecordingSweep(RecordingOptions{}); len(res.Expired) != 1 { + t.Fatalf("retention_days 1: %+v", res) + } +} + +func TestRecordingIdleAutoFinalize(t *testing.T) { + svc, _, clock := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(4)); err != nil { + t.Fatal(err) + } + empty := startRecording(t, svc, "ben", "audio/webm") + clock.t = clock.t.Add(time.Hour) + if res, _ := svc.RecordingSweep(RecordingOptions{}); len(res.Finalized)+len(res.Removed) != 0 { + t.Fatalf("finalized before idle: %+v", res) + } + clock.t = clock.t.Add(time.Hour) + res, err := svc.RecordingSweep(RecordingOptions{}) + if err != nil || len(res.Finalized) != 1 || res.Finalized[0].ID != m.ID || !res.Finalized[0].AutoFinalized || len(res.Removed) != 1 || res.Removed[0].ID != empty.ID { + t.Fatalf("idle sweep = %+v, %v", res, err) + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].ID != m.ID { + t.Fatalf("items after auto-finalize = %+v", items) + } + // The user may record again once the old one is finalized. + startRecording(t, svc, "ana", "audio/webm") +} + +func TestRecordingSourceSurvivesSetSources(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + svc.SetSources(nil) + if !svc.HasSource(RecordingType) { + t.Fatal("SetSources dropped the recording source") + } + if !svc.Settings().Recording.Enabled || svc.Settings().Recording.ConsentVersion != RecordingConsentVersion || svc.Settings().Recording.MaxChunkBytes != RecordingMaxChunkBytes { + t.Fatalf("settings = %+v", svc.Settings().Recording) + } +} + +func TestRecordingFetchFromFact(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(4)); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); err != nil { + t.Fatal(err) + } + it, ok, _ := svc.Find(context.Background(), RecordingType, m.ID) + if !ok { + t.Fatal("finalized recording not found") + } + c, err := svc.Content(context.Background(), it) + if err != nil || !c.Pending || c.Provenance != RecordingType+"://recordings/"+m.ID { + t.Fatalf("content = %+v, %v", c, err) + } + // Without a fact (write failed) the item still reads as pending. + c, err = (&RecordingSource{Store: NewRecordingStore(t.TempDir())}).Fetch(context.Background(), Item{ID: "x", Title: "t"}) + if err != nil || !c.Pending { + t.Fatalf("fact-less content = %+v, %v", c, err) + } + f, a, err := (&RecordingSource{Store: NewRecordingStore(t.TempDir())}).Store.OpenAudio("nope") + if f != nil || a.ID != "" || !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("open unknown = %v", err) + } +} + +func TestSniffRecordingContainer(t *testing.T) { + for _, tc := range []struct { + head []byte + want string + }{ + {webmChunk(0), RecordingContainerWebM}, + {mp4Box("ftyp", []byte("M4A ")), RecordingContainerMP4}, + {[]byte("ID3\x04"), ""}, + {[]byte("OggS\x00\x02"), ""}, + {[]byte{0x1A, 0x45}, ""}, + {nil, ""}, + } { + if got := SniffRecordingContainer(tc.head); got != tc.want { + t.Errorf("sniff %q = %q, want %q", tc.head, got, tc.want) + } + } + if RecordingContainer("audio/mp4; codecs=\"mp4a.40.2\"") != RecordingContainerMP4 || RecordingContainer("audio/webm;codecs=opus") != RecordingContainerWebM { + t.Fatal("MediaRecorder MIME parameters not accepted") + } +} diff --git a/src/pkg/inbox/recordingsource.go b/src/pkg/inbox/recordingsource.go new file mode 100644 index 0000000000..09cfca495b --- /dev/null +++ b/src/pkg/inbox/recordingsource.go @@ -0,0 +1,394 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +const ( + // RecordingLayer is the Knowledge layer recording facts are written to: + // personal, so a recording is never published to the hub. + RecordingLayer = "personal" + // recordingAudioPathPrefix is the same-origin audio endpoint. + recordingAudioPathPrefix = "/api/inbox/recordings/" + // recordingAudioLinkTTL is how long a same-origin audio link is quoted + // as valid (it needs the dashboard session either way). + recordingAudioLinkTTL = time.Hour + // RecordingSweepActor is the audit actor of sweep-made changes. + RecordingSweepActor = "inbox-recording-sweep" +) + +// LocalAudio is an audio file on this hive's own disk. +type LocalAudio struct { + File *os.File + Name string + MIME string + Size int64 + ModTime time.Time +} + +// AudioOpener is optionally implemented by sources whose audio lives on this +// hive: playback serves the file directly, and nothing makes an HTTP call to +// the hive itself. +type AudioOpener interface { + OpenAudio(ctx context.Context, it Item) (LocalAudio, error) +} + +// RecordingSource lists the recordings made in this hive's dashboard. It is +// a builtin source (listed whatever inbox.sources says). Each finalized +// recording has a Knowledge fact (personal layer) that starts pending and +// receives the machine transcript; the raw audio is deleted after +// inbox.recording.retention_days while the fact and transcript stay. +type RecordingSource struct { + Store *RecordingStore + // FactDir is /connectors/. + FactDir string + // RetentionDays reports inbox.recording.retention_days for listings. + RetentionDays func() int +} + +// Type implements Source. +func (r *RecordingSource) Type() string { return RecordingType } + +// Builtin implements BuiltinSource. +func (r *RecordingSource) Builtin() bool { return true } + +// factPath is where recording id's fact lives. +func (r *RecordingSource) factPath(id string) string { + return filepath.Join(r.FactDir, connector.Slug(RecordingType, RecordingConnector, id)+".md") +} + +// List implements Source: finalized recordings, newest first. +func (r *RecordingSource) List(ctx context.Context) ([]Item, error) { + days := config.DefaultInboxRecordingRetentionDays + if r.RetentionDays != nil { + days = r.RetentionDays() + } + var out []Item + for _, m := range r.Store.List() { + if ctx.Err() != nil { + return out, ctx.Err() + } + if m.State != RecordingFinalized { + continue + } + it := Item{ + Source: RecordingType, Connector: RecordingConnector, ID: m.ID, Title: m.Title, + RecordedAt: m.CreatedAt, DurationMS: m.DurationMS, Tags: []string{"recording"}, + RecordedBy: m.RecordedBy, AudioExpired: m.AudioExpired(), AudioRetentionDays: days, + Pending: true, FactState: connector.FactContentPending, + } + if m.IncludesTabAudio { + it.Tags = append(it.Tags, "tab-audio") + } + path := r.factPath(m.ID) + if data, err := os.ReadFile(path); err == nil { + fm, body := splitFrontMatter(string(data)) + fact := itemFromFact(fm, body, RecordingLayer, path) + it.FactSlug, it.FactPath, it.Layer, it.FactState = fact.FactSlug, fact.FactPath, fact.Layer, fact.FactState + it.Summary, it.HasTranscript, it.Pending, it.Excerpt = fact.Summary, fact.HasTranscript, fact.Pending, fact.Excerpt + } + out = append(out, it) + } + sortItems(out) + return out, nil +} + +// Fetch implements Source from the recording's fact. +func (r *RecordingSource) Fetch(ctx context.Context, it Item) (Content, error) { + provenance := RecordingType + "://recordings/" + it.ID + if it.FactPath == "" { + return Content{Title: it.Title, Pending: true, Provenance: provenance}, nil + } + c, err := (&FactSource{SourceType: RecordingType}).Fetch(ctx, it) + if err != nil && errors.Is(err, os.ErrNotExist) { + return Content{Title: it.Title, Pending: true, Provenance: provenance}, nil + } + return c, err +} + +// AudioLink implements AudioSource with the hive's own same-origin audio +// endpoint; there is no third-party storage behind a recording. +func (r *RecordingSource) AudioLink(_ context.Context, it Item) (AudioLink, error) { + m, ok := r.Store.Get(it.ID) + if !ok || m.State != RecordingFinalized { + return AudioLink{}, ErrNoAudio + } + if m.AudioExpired() { + return AudioLink{}, fmt.Errorf("%w: %v", ErrNoAudio, ErrRecordingExpired) + } + return AudioLink{URL: recordingAudioPathPrefix + m.ID + "/audio", ExpiresAt: time.Now().Add(recordingAudioLinkTTL).UTC()}, nil +} + +// OpenAudio implements AudioOpener. +func (r *RecordingSource) OpenAudio(_ context.Context, it Item) (LocalAudio, error) { + f, m, err := r.Store.OpenAudio(it.ID) + if err != nil { + return LocalAudio{}, err + } + st, err := f.Stat() + if err != nil { + _ = f.Close() + return LocalAudio{}, err + } + return LocalAudio{File: f, Name: m.AudioFile, MIME: recordingAudioMIME(m.Container), Size: st.Size(), ModTime: st.ModTime()}, nil +} + +// recordingAudioMIME is the Content-Type a recording is served with. +func recordingAudioMIME(container string) string { + if container == RecordingContainerMP4 { + return "audio/mp4" + } + return "audio/webm" +} + +// writeFact writes the pending Knowledge fact of a finalized recording. A +// fact that already holds a machine transcript is left alone. +func (r *RecordingSource) writeFact(m RecordingMeta) error { + if r.FactDir == "" { + return nil + } + if err := os.MkdirAll(r.FactDir, recordingDirMode); err != nil { + return err + } + dur := time.Duration(m.DurationMS) * time.Millisecond + var md strings.Builder + md.WriteString("# " + m.Title + "\n\n") + md.WriteString("- **Recorded:** " + m.CreatedAt.UTC().Format(time.RFC3339) + "\n") + md.WriteString("- **Recorded by:** " + m.RecordedBy + "\n") + md.WriteString("- **Duration:** " + dur.Round(time.Second).String() + "\n") + if m.IncludesTabAudio { + md.WriteString("- **Audio:** microphone and a browser tab\n") + } else { + md.WriteString("- **Audio:** microphone\n") + } + md.WriteString("\n_" + RecordingPendingText + "_\n") + w := &connector.FactWriter{Dir: r.FactDir} + _, _, err := w.Write(connector.ConnectorConfig{Name: RecordingConnector, Type: RecordingType, Layer: RecordingLayer}, connector.Page{ + ID: m.ID, + Title: m.Title, + Attrs: map[string]string{ + "hive_recording_content": connector.FactContentPending, + "recorded_at": m.CreatedAt.UTC().Format(time.RFC3339), + "duration_ms": strconv.FormatInt(m.DurationMS, 10), + "recorded_by": m.RecordedBy, + "consent_version": m.ConsentVersion, + "includes_tab_audio": strconv.FormatBool(m.IncludesTabAudio), + "tags": "recording", + }, + Markdown: md.String(), + UpdatedAt: m.CreatedAt, + }) + return err +} + +// RecordingSettings is the dashboard's view of inbox.recording. +type RecordingSettings struct { + Enabled bool `json:"enabled"` + ConsentVersion string `json:"consent_version"` + RetentionDays int `json:"retention_days"` + MaxSeconds int `json:"max_seconds"` + MaxBytes int64 `json:"max_bytes"` + MaxChunkBytes int `json:"max_chunk_bytes"` + QuotaBytes int64 `json:"quota_bytes"` + UsedBytes int64 `json:"used_bytes"` + ChunkMillis int `json:"chunk_ms"` + BitsPerSecond int `json:"audio_bits_per_second"` + // Transcribes reports that a finished recording is transcribed + // automatically right now (transcription enabled with consent). + Transcribes bool `json:"transcribes"` +} + +// RecordingAudit is one recording audit entry. +type RecordingAudit struct { + Actor string + Action string + Fields []string +} + +// Detail renders the entry as "key=value, key=value". +func (a RecordingAudit) Detail() string { + parts := make([]string, 0, len(a.Fields)/2) + for i := 0; i+1 < len(a.Fields); i += 2 { + parts = append(parts, a.Fields[i]+"="+a.Fields[i+1]) + } + return strings.Join(parts, ", ") +} + +// RecordingOptions carries who acts and where audit entries go. +type RecordingOptions struct { + Actor string + Audit func(RecordingAudit) +} + +func (o RecordingOptions) audit(action string, fields ...string) { + if o.Audit != nil { + o.Audit(RecordingAudit{Actor: o.Actor, Action: action, Fields: fields}) + } +} + +// EnableRecordings wires dashboard recordings; the source is registered as +// a builtin and survives SetSources. +func (s *Service) EnableRecordings(src *RecordingSource) { + s.mu.Lock() + s.recordings = src + if src != nil { + s.sources[src.Type()] = src + } + s.mu.Unlock() +} + +// recordingSource returns the wired recording source, or nil. +func (s *Service) recordingSource() *RecordingSource { + s.mu.RLock() + defer s.mu.RUnlock() + return s.recordings +} + +// RecordingSettings returns inbox.recording with defaults applied. +func (s *Service) RecordingSettings() RecordingSettings { + c := s.Config() + r := c.Recording + out := RecordingSettings{ + Enabled: r.EffectiveEnabled() && s.recordingSource() != nil, ConsentVersion: RecordingConsentVersion, + RetentionDays: r.EffectiveRetentionDays(), MaxSeconds: r.EffectiveMaxSeconds(), MaxBytes: r.EffectiveMaxBytes(), + MaxChunkBytes: RecordingMaxChunkBytes, QuotaBytes: r.EffectiveQuotaBytes(), ChunkMillis: RecordingChunkMillis, + BitsPerSecond: RecordingAudioBitsPerSecond, Transcribes: c.Transcription.Active(), + } + if src := s.recordingSource(); src != nil { + out.UsedBytes = src.Store.UsedBytes() + } + return out +} + +// recordingStore returns the store when recording is enabled. +func (s *Service) recordingStore() (*RecordingSource, config.InboxRecording, error) { + src := s.recordingSource() + cfg := s.Config().Recording + if src == nil || !cfg.EffectiveEnabled() { + return nil, cfg, ErrRecordingDisabled + } + return src, cfg, nil +} + +// StartRecording creates a recording for req.User. +func (s *Service) StartRecording(req RecordingStart, opts RecordingOptions) (RecordingMeta, error) { + src, cfg, err := s.recordingStore() + if err != nil { + return RecordingMeta{}, err + } + m, err := src.Store.Create(req, cfg) + if err != nil { + return RecordingMeta{}, err + } + opts.audit("inbox_recording_start", "recording", m.ID, "consent_version", m.ConsentVersion, "includes_tab_audio", strconv.FormatBool(m.IncludesTabAudio), "mime", m.MIME) + return m, nil +} + +// AppendRecordingChunk stores chunk seq of one of user's recordings. +func (s *Service) AppendRecordingChunk(id, user string, seq int, data []byte) (RecordingChunk, error) { + src, cfg, err := s.recordingStore() + if err != nil { + return RecordingChunk{}, err + } + return src.Store.Append(id, user, seq, data, cfg) +} + +// FinalizeRecording ends one of user's recordings and writes its pending +// Knowledge fact. +func (s *Service) FinalizeRecording(id, user string, durationMS int64, opts RecordingOptions) (RecordingMeta, error) { + src, cfg, err := s.recordingStore() + if err != nil { + return RecordingMeta{}, err + } + m, err := src.Store.Finalize(id, user, durationMS, cfg) + if err != nil { + return RecordingMeta{}, err + } + s.recordingFinalized(src, m, opts) + return m, nil +} + +// recordingFinalized writes the fact and audits a finished recording. +func (s *Service) recordingFinalized(src *RecordingSource, m RecordingMeta, opts RecordingOptions) { + if err := src.writeFact(m); err != nil { + s.Logger.Warn("inbox: write recording fact", "recording", m.ID, "error", err) + } + if m.ConcatWarning != "" { + s.Logger.Warn("inbox: recording container check", "recording", m.ID, "warning", m.ConcatWarning) + } + opts.audit("inbox_recording_finalize", "recording", m.ID, "chunks", strconv.Itoa(m.Chunks), "bytes", strconv.FormatInt(m.Bytes, 10), + "duration_ms", strconv.FormatInt(m.DurationMS, 10), "auto", strconv.FormatBool(m.AutoFinalized)) +} + +// AbortRecording discards an in-progress recording. +func (s *Service) AbortRecording(id, user string, asOwner bool, opts RecordingOptions) (RecordingMeta, error) { + src := s.recordingSource() + if src == nil { + return RecordingMeta{}, ErrRecordingDisabled + } + m, err := src.Store.Abort(id, user, asOwner) + if err != nil { + return RecordingMeta{}, err + } + opts.audit("inbox_recording_abort", "recording", m.ID, "recorded_by", m.RecordedBy, "chunks", strconv.Itoa(m.Chunks)) + return m, nil +} + +// OpenRecordingAudio opens a finalized recording's audio (playback works +// while recording is disabled: the recordings stay until retention). +func (s *Service) OpenRecordingAudio(id string) (LocalAudio, error) { + src := s.recordingSource() + if src == nil { + return LocalAudio{}, ErrRecordingNotFound + } + return src.OpenAudio(context.Background(), Item{Source: RecordingType, ID: id}) +} + +// OpenLocalAudio opens an item's audio when its source keeps it on this +// hive; ok is false for sources whose audio lives elsewhere. +func (s *Service) OpenLocalAudio(ctx context.Context, it Item) (LocalAudio, bool, error) { + op, ok := s.source(it.Source).(AudioOpener) + if !ok { + return LocalAudio{}, false, nil + } + a, err := op.OpenAudio(ctx, it) + return a, true, err +} + +// RecordingSweep finalizes idle recordings and deletes audio past +// retention (it runs periodically from boot). Every change is logged and +// audited with RecordingSweepActor. +func (s *Service) RecordingSweep(opts RecordingOptions) (RecordingSweepResult, error) { + src := s.recordingSource() + if src == nil { + return RecordingSweepResult{}, nil + } + if opts.Actor == "" { + opts.Actor = RecordingSweepActor + } + cfg := s.Config().Recording + res, err := src.Store.Sweep(cfg) + for _, m := range res.Finalized { + s.Logger.Info("inbox: recording auto-finalized after idle", "recording", m.ID, "bytes", m.Bytes) + s.recordingFinalized(src, m, opts) + } + for _, m := range res.Removed { + s.Logger.Info("inbox: empty idle recording removed", "recording", m.ID) + opts.audit("inbox_recording_abort", "recording", m.ID, "recorded_by", m.RecordedBy, "chunks", "0", "reason", "idle") + } + for _, m := range res.Expired { + s.Logger.Info("inbox: recording audio deleted (retention)", "recording", m.ID, "retention_days", cfg.EffectiveRetentionDays(), "bytes", m.Bytes) + opts.audit("inbox_recording_audio_expired", "recording", m.ID, "retention_days", strconv.Itoa(cfg.EffectiveRetentionDays()), "bytes", strconv.FormatInt(m.Bytes, 10)) + } + return res, err +} diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go index 01afbc88ff..2f7e10c864 100644 --- a/src/pkg/inbox/service.go +++ b/src/pkg/inbox/service.go @@ -35,6 +35,8 @@ type Settings struct { // Uploads are the drop-zone limits; Enabled is false when this hive has // no upload store. Uploads UploadSettings `json:"uploads"` + // Recording is inbox.recording with defaults applied. + Recording RecordingSettings `json:"recording"` } // Service lists items across sources, applies persisted state, runs @@ -94,6 +96,9 @@ type Service struct { // uploads holds files and links added on the dashboard (see // upload_source.go); nil until EnableUploads. uploads *UploadSource + // recordings is the dashboard recording source (see recordingsource.go); + // nil until EnableRecordings. + recordings *RecordingSource } // NewService builds a service for cfg with the given sources/destinations. @@ -123,6 +128,11 @@ func (s *Service) SetSources(sources []Source) { if s.uploads != nil { next[UploadType] = s.uploads } + if s.recordings != nil { + if _, ok := next[RecordingType]; !ok { + next[RecordingType] = s.recordings + } + } s.sources = next s.mu.Unlock() } @@ -210,6 +220,7 @@ func (s *Service) Settings() Settings { if u := s.Uploads(); u != nil { st.Uploads = u.Settings() } + st.Recording = s.RecordingSettings() return st } diff --git a/src/pkg/inbox/transcription.go b/src/pkg/inbox/transcription.go index 175301e0ac..a0a2781207 100644 --- a/src/pkg/inbox/transcription.go +++ b/src/pkg/inbox/transcription.go @@ -197,7 +197,7 @@ func (s *Service) transcriber(t config.InboxTranscription) (transcribe.Transcrib // hasAudio reports whether the item's source can mint audio links. func (s *Service) hasAudio(it Item) bool { _, ok := s.source(it.Source).(AudioSource) - return ok + return ok && !it.AudioExpired } // Reason codes for Item.TranscribeUnavailable. no_key means the resolved