diff --git a/changelog.d/added-inbox-upload-media.md b/changelog.d/added-inbox-upload-media.md new file mode 100644 index 0000000000..6a7b7fa425 --- /dev/null +++ b/changelog.d/added-inbox-upload-media.md @@ -0,0 +1 @@ +- Drop audio and video onto the Inbox (mp3, m4a, aac, wav, ogg, opus, webm, mp4, mov, checked by container signature): the file streams straight to the hive's disk under the recording size cap (`inbox.recording.max_bytes`, 200 MiB), plays back from disk through the same-origin stream, and is transcribed as soon as it is added when transcription is enabled (otherwise 🎧 Transcribe shows why it is unavailable); the machine transcript lands in the item's Knowledge fact. Originals are deleted after the recordings retention (5 days, `inbox.uploads.media_retention_days` overrides) while the transcript and item stay; video keeps only its audio track. Uploads now get a 20-minute read window instead of the dashboard's 30-second read timeout. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index bf04b17dda..4ab08794e2 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -40173,7 +40173,7 @@ "Knowledge" ], "summary": "Add dropped files to the Inbox", - "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV, TSV, HTML, Word (.docx, read with zip-bomb caps and no DTDs or external entities) and PDF (parsed out of process in the hive-pdftext sandbox; a PDF without a text layer is refused) become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", + "description": "Owner or read-write. Each multipart `file` field is typed by its content (magic bytes, UTF-8 validation and a parse check), never by its name or client MIME type alone. Markdown, text, JSON, JSON Lines, YAML, CSV, TSV, HTML, Word (.docx, read with zip-bomb caps and no DTDs or external entities) and PDF (parsed out of process in the hive-pdftext sandbox; a PDF without a text layer is refused) become; audio and video (mp3, m4a, aac, wav, ogg, opus, webm, mp4, mov, checked by container signature) are streamed to disk under `inbox.recording.max_bytes`, played back from disk through the same-origin stream and transcribed when transcription is on, and their originals are deleted after the recordings retention. Upload items otherwise become `upload` Inbox items with extracted markdown, stored under `connector-state/uploads//` (0700 directory, 0600 files) and written as a Knowledge fact with the text inside an `` envelope. Limits come from `inbox.uploads` (`max_file_bytes`, default 25 MiB; `max_files_per_drop`, default 20; `quota_bytes`, default 2 GiB). Requests marked `Sec-Fetch-Site: cross-site` are refused. Every file is audited as `inbox_upload` with its outcome.", "requestBody": { "required": true, "content": { @@ -40236,7 +40236,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40304,7 +40306,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40372,7 +40376,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40446,7 +40452,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40514,7 +40522,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40582,7 +40592,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -40650,7 +40662,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { @@ -41421,7 +41435,9 @@ "tsv", "html", "docx", - "pdf" + "pdf", + "audio", + "video" ] }, "code": { diff --git a/dashboard/test/inbox-item.test.js b/dashboard/test/inbox-item.test.js index 83f46a4c24..39e0a8c860 100644 --- a/dashboard/test/inbox-item.test.js +++ b/dashboard/test/inbox-item.test.js @@ -184,3 +184,13 @@ test('share history pills sit with the other pills and read users get no link', assert.match(html, /]*>→ RFC #18412/); assert.doesNotMatch(html, /href="https:\/\/github/); }); + +test('an uploaded audio file is pending with its own note and keeps the player', () => { + const ctx = makeContext('read-write'); + const audio = { source: 'upload', id: '20261116t093000-0a1b2c3e', title: 'standup', origin: 'upload · standup.m4a', added_by: 'ana', upload_kind: 'audio', pending: true, transcribable: true, fact_slug: 'upload-x', fact_state: 'pending' }; + const html = vm.runInContext('inboxItemHtml(' + JSON.stringify(audio) + ')', ctx); + assert.match(html, /this audio was added to the Inbox without text/); + assert.doesNotMatch(html, /Plaud/); + assert.match(html, /data-action="inboxPlay"/); + assert.match(html, /▸ Transcript<\/button>/); +}); diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index e1526c6180..95865533b5 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -76,8 +76,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/inbox/transcriptions` | Dashboard auth | Live progress of queued and running transcriptions in queue order (`{active: [{source, id, state}]}` with `phase`, `progress`, `estimated`, `started_at`, `audio_seconds`, `processed_seconds`, `eta_seconds`, `queue_position`); reads no source, so the Inbox polls it every 3 s while a job runs; `Cache-Control: no-store`; see [inbox-transcription.md](../inbox-transcription.md) | `pkg/dashboard/api_inbox.go:34` | | `GET` | `/api/inbox/{source}/{id}/audio` | Owner or read-write | Mint a fresh short-lived audio/video link for one item (`{url, expires_at}`; Plaud: presigned, 24h; Zoom: audio-only or video `download_url` with a one-hour download token; Webex: the recording's temporary direct audio (else MP4) download link, valid 3 hours; Teams and Google Meet: 404, no browser-playable link; Slack huddles: 404, Slack keeps no huddle audio; Otter: the conversation's MP3 download link (`GET /conversations/{id}/audio`), minted per click; Granola: 404, Granola keeps no meeting audio; Limitless: 404, Pendant audio is served only behind the API key; Fathom: the signed url of a recording download (`POST /recordings/{id}/download`, audio-only recordings complete at once, video is polled), valid about 24 hours; Fireflies: the transcript's hashed `audio_url` (else `video_url`), valid 24 hours, Pro plan or higher); never stored by the hive, replied with `Cache-Control: no-store`, audited as `inbox_audio` (403 role, 404 unknown item or no audio, 502 source unreachable) | `pkg/dashboard/api_inbox.go:30` | | `GET` | `/api/inbox/{source}/{id}/audio/stream` | Owner or read-write | Stream one item's audio from the dashboard origin, for the Inbox player (the CSP refuses cross-origin media). The hive mints the link as above, fetches it server-side and copies the bytes with the media `Content-Type` (from storage, else the link's extension); a single `Range: bytes=…` is passed through (206 + `Content-Range`; 200 whole file when storage ignores it; 416 passed through). The link stays in memory for one playback's Range requests (10 min, never past expiry), is never logged, returned or stored, and is re-minted once when storage refuses it; nothing is written to disk. `Cache-Control: no-store`; audited as `inbox_audio_stream` once per minted link (403 role, 404 unknown item or no audio, 413 over the 2 GiB cap, 502 source or storage unreachable, or link refused/expired) | `pkg/dashboard/api_inbox.go:31` | -| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv, html, docx and pdf (parsed in the `hive-pdftext` sandbox) become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | -| `POST` | `/api/inbox/uploads/url` | Owner or read-write | Add a link: `{url}`. The hive fetches it under the SSRF guard (http/https only, no credentials, its own DNS lookup with the connection pinned to the checked address, private/loopback/link-local/CGNAT/multicast/ULA/metadata ranges refused, every redirect re-checked, at most 3, body capped at `inbox.uploads.max_file_bytes`, 60 s) and runs the body through the upload type pipeline (an HTML page becomes readable markdown titled by its ``). 201 `{results: [{filename: host, ok, source, item_id, title, kind}]}`; 400 `link_refused`, 413, 415, 422, 502 `fetch_failed`, 507; audited as `inbox_upload` with `origin=link` and the host (403 role, 404 no upload store) | `pkg/dashboard/api_inbox_uploads.go:28` | +| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv, html, docx and pdf (parsed in the `hive-pdftext` sandbox) become `upload` items; audio and video (mp3, m4a, aac, wav, ogg, opus, webm, mp4, mov) stream to disk under `inbox.recording.max_bytes`, become pending audio items played from disk and are transcribed when transcription is on with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:34` | +| `POST` | `/api/inbox/uploads/url` | Owner or read-write | Add a link: `{url}`. The hive fetches it under the SSRF guard (http/https only, no credentials, its own DNS lookup with the connection pinned to the checked address, private/loopback/link-local/CGNAT/multicast/ULA/metadata ranges refused, every redirect re-checked, at most 3, body capped at `inbox.uploads.max_file_bytes`, 60 s) and runs the body through the upload type pipeline (an HTML page becomes readable markdown titled by its `<title>`). 201 `{results: [{filename: host, ok, source, item_id, title, kind}]}`; 400 `link_refused`, 413, 415, 422, 502 `fetch_failed`, 507; audited as `inbox_upload` with `origin=link` and the host (403 role, 404 no upload store) | `pkg/dashboard/api_inbox_uploads.go:35` | | `POST` | `/api/inbox/recordings` | Owner or read-write | Start a dashboard recording (● Record): `{title, mime: audio/webm\|audio/mp4, consent_version, includes_tab_audio}`. `consent_version` must name the current consent notice; it is stored with the user in the recording's metadata. One recording per user at a time. 201 `{id, max_chunk_bytes, max_seconds, retention_days, transcribes}`; audited as `inbox_recording_start` (400 consent missing or stale, 403 role, 409 `{active_id}` already recording, 412 `inbox.recording.enabled` off, 415 other media types, 507 `inbox.recording.quota_bytes` used up); see [inbox-recording.md](../inbox-recording.md) | `pkg/dashboard/api_inbox_recordings.go:20` | | `POST` | `/api/inbox/recordings/{id}/chunks` | The recorder (owner or read-write) | Store one MediaRecorder chunk `?seq=N`, the raw bytes as the body (at most `max_chunk_bytes`, 512 KiB). In order only; each part is fsynced before the 200 `{id, seq, next_seq, bytes, duplicate?}`. The first chunk must start with a WebM or MP4 signature; the hive never decodes audio. A re-send with the same bytes is acknowledged again (`duplicate: true`) (400 bad seq, 403 someone else's, 409 `{expected_seq}` gap, other bytes for a stored seq, or already ended, 413 chunk over 512 KiB or past `max_bytes` / `max_seconds`, 415 not WebM/MP4, 507 quota) | `pkg/dashboard/api_inbox_recordings.go:21` | | `POST` | `/api/inbox/recordings/{id}/finalize` | The recorder | End a recording: `{duration_ms?}` (the recorder's count, pauses excluded). The chunks are concatenated in order into `audio.webm` or `audio.mp4`, a pending Knowledge fact (personal layer) is written and the item appears in the Inbox. 200 `{id, state, title, duration_ms, bytes, chunks, transcribes}`; audited as `inbox_recording_finalize` (400 no chunk received, 403, 404, 409 already ended) | `pkg/dashboard/api_inbox_recordings.go:22` | diff --git a/src/docs/inbox-uploads.md b/src/docs/inbox-uploads.md index 714a8e25f8..543d7581f6 100644 --- a/src/docs/inbox-uploads.md +++ b/src/docs/inbox-uploads.md @@ -32,6 +32,8 @@ and opens as a Word document. | CSV / TSV | `.csv`, `.tsv`, `.tab` | row and column counts and a markdown table of the first 20 rows (20 columns at most) | | Word | `.docx` | headings (Title and Heading 1 to 6 styles) as `#`, paragraphs, bulleted and numbered lists (by level) as `-` items, and tables as markdown tables; deleted text and field codes are left out; the title comes from the document properties, else the first heading. Read in Go with `archive/zip` and `encoding/xml` (standard library): at most 1000 zip entries, 200 MiB declared uncompressed in all, 50 MiB per part (checked while inflating, not only from the header) and a 100:1 compression ratio per entry; a part with a DOCTYPE or an entity the decoder does not know is refused (XXE), and nothing is fetched | | PDF | `.pdf` (or a file without an extension that starts with `%PDF-`) | one `## Page N` section per page with text, a page-count summary, and the document title from its Info dictionary. Parsed **out of process** in the `hive-pdftext` sandbox with `github.com/ledongthuc/pdf` (BSD-3-Clause, pure Go): at most 200 pages and 1 MiB of text; a PDF without a text layer is refused with "no text layer (scanned PDF); OCR not supported yet" | +| Audio | `.mp3`, `.m4a`, `.aac`, `.wav`, `.ogg`, `.opus`, `.webm` | an audio item with ▶ Play through the same-origin stream (served from this hive's disk) and 🎧 Transcribe; when transcription is enabled and consented, the hive queues the transcription as soon as the file is added. The content must carry the container's signature (ID3 or MPEG frame sync, ADTS, `ftyp`, RIFF/WAVE, Ogg, EBML) | +| Video | `.mp4`, `.mov` | the same as audio: only the audio track is used (for transcription and playback); there is no video playback | | HTML | `.html`, `.htm`, and web pages behind a link | readable markdown: the `<title>` names the item; the `<article>` or `<main>` (else `<body>`) is kept with headings, paragraphs, lists, quotes, code, tables, link targets and image alt text; scripts, styles, forms, navigation, footers, asides and hidden elements are dropped. Parsed with `golang.org/x/net/html` (BSD-3-Clause); nothing is executed and nothing the page references is loaded | Text is validated as UTF-8 (a UTF-8 BOM is dropped; UTF-16 is refused with @@ -47,7 +49,6 @@ Not supported (the item is refused with a message saying what to do): | RTF, OpenDocument `.odt` | save it as .docx, PDF or plain text | | PowerPoint `.pptx` | export the slides as PDF | | Excel `.xlsx` | export the sheet as CSV | -| Audio and video | not supported yet | | Executables, archives, images, anything else | unsupported file type | ## What happens to a file @@ -72,6 +73,25 @@ Every file, accepted or refused, is audited as `inbox_upload` with the user, the item id, the detected kind, the size and the outcome or refusal code; the file's content and name are not written to the audit log. +## Audio and video + +An audio or video file is streamed straight to `connector-state/uploads/<id>/original` +(never held in memory), capped by `inbox.recording.max_bytes` and the uploads +quota, and its item starts **pending**: it has no text until it is +transcribed. ▶ Play streams it from this hive's disk through the same +same-origin endpoint recordings use (`GET /api/inbox/upload/<id>/audio/stream`, +with Range for seeking), and transcription reads the same file locally, so +with whisper the audio never leaves the hive. When `inbox.transcription` is +enabled and consented, the transcription is queued as soon as the file is +added (audited as `inbox_transcribe`, actor = who added it); otherwise the +item shows the usual reason 🎧 Transcribe is unavailable. The machine +transcript is written into the item's Knowledge fact like any recording's. +The original is deleted after `inbox.uploads.media_retention_days`, which +defaults to `inbox.recording.retention_days` (5 days); the transcript, the +item and the fact stay. Video files keep only their audio track: there is +no video playback. The browser request may take up to 20 minutes to arrive +(the dashboard's 30-second read timeout is lifted for uploads). + ## PDF sandbox PDF parsers have a long history of bugs, so the hive never parses an @@ -141,7 +161,8 @@ through). | `inbox.uploads.max_files_per_drop` | `HIVE_INBOX_UPLOADS_MAX_FILES_PER_DROP` | `20` | files in one drop or one request | | `inbox.uploads.quota_bytes` | `HIVE_INBOX_UPLOADS_QUOTA_BYTES` | `2147483648` (2 GiB) | everything stored under `connector-state/uploads` | | `inbox.uploads.document_retention_days` | `HIVE_INBOX_UPLOADS_DOCUMENT_RETENTION_DAYS` | `0` (keep) | delete a document's original after this many days | -| `inbox.uploads.media_retention_days` | `HIVE_INBOX_UPLOADS_MEDIA_RETENTION_DAYS` | `5` | delete an audio or video original after this many days | +| `inbox.uploads.media_retention_days` | `HIVE_INBOX_UPLOADS_MEDIA_RETENTION_DAYS` | `inbox.recording.retention_days` (5) | delete an audio or video original after this many days | +| `inbox.recording.max_bytes` | | `209715200` (200 MiB) | largest audio or video file (the recording cap) | ```yaml inbox: diff --git a/src/pkg/dashboard/api_inbox_uploads.go b/src/pkg/dashboard/api_inbox_uploads.go index d7eca3b5fe..3ede1749bb 100644 --- a/src/pkg/dashboard/api_inbox_uploads.go +++ b/src/pkg/dashboard/api_inbox_uploads.go @@ -1,6 +1,7 @@ package dashboard import ( + "bufio" "context" "errors" "io" @@ -20,6 +21,12 @@ const ( inboxUploadOverheadBytes = 64 << 10 // inboxUploadTimeout bounds one upload request's extraction and storage. inboxUploadTimeout = 2 * time.Minute + // inboxUploadReadWindow replaces the dashboard's 30-second read timeout + // for an upload body: a 25 MB document or a 200 MB recording on a slow + // link needs longer to arrive. inboxUploadMediaTimeout bounds a request + // that carries audio or video. + inboxUploadReadWindow = 20 * time.Minute + inboxUploadMediaTimeout = 25 * time.Minute ) // registerInboxUploadRoutes adds the drag-and-drop ingest routes. @@ -100,13 +107,17 @@ func (s *Server) handleInboxUpload(w http.ResponseWriter, r *http.Request) { } lim := svc.Config().Uploads maxFile, maxFiles := lim.EffectiveMaxFileBytes(), lim.EffectiveMaxFilesPerDrop() - r.Body = http.MaxBytesReader(w, r.Body, maxFile*int64(maxFiles)+inboxUploadOverheadBytes) + // Audio and video stream to disk under the recording cap. + maxMedia := svc.Config().Recording.EffectiveMaxBytes() + r.Body = http.MaxBytesReader(w, r.Body, max(maxFile, maxMedia)*int64(maxFiles)+inboxUploadOverheadBytes) + // Best effort: a server without per-request deadlines keeps its own. + _ = http.NewResponseController(w).SetReadDeadline(time.Now().Add(inboxUploadReadWindow)) mr, err := r.MultipartReader() if err != nil { jsonError(w, "expected a multipart upload with one or more \"file\" fields", http.StatusBadRequest) return } - ctx, cancel := context.WithTimeout(r.Context(), inboxUploadTimeout) + ctx, cancel := context.WithTimeout(r.Context(), inboxUploadMediaTimeout) defer cancel() user := requestUser(r) var results []InboxUploadResult @@ -133,7 +144,16 @@ func (s *Server) handleInboxUpload(w http.ResponseWriter, r *http.Request) { results = append(results, s.inboxUploadRefused(r, name, 0, inbox.UploadErrTooMany, "too many files in one drop: at most "+strconv.Itoa(maxFiles))) continue } - data, err := io.ReadAll(io.LimitReader(part, maxFile+1)) + br := bufio.NewReaderSize(part, inbox.MediaSniffBytes) + head, _ := br.Peek(inbox.MediaSniffBytes) + if _, media, uerr := inbox.SniffMedia(name, head); uerr != nil { + results = append(results, s.inboxUploadRefused(r, name, 0, uerr.Code, uerr.Message)) + continue + } else if media { + results = append(results, s.inboxUploadMedia(ctx, r, svc, part.FileName(), name, br, len(head))) + continue + } + data, err := io.ReadAll(io.LimitReader(br, maxFile+1)) if err != nil { if isUploadTooLarge(err) { jsonError(w, "the upload is too large", http.StatusRequestEntityTooLarge) @@ -182,6 +202,37 @@ func (s *Server) handleInboxUpload(w http.ResponseWriter, r *http.Request) { jsonStatusResponse(w, code, InboxUploadResponse{Results: results}) } +// inboxUploadMedia streams one audio or video part into the store and +// queues its transcription when transcription is on. +func (s *Server) inboxUploadMedia(ctx context.Context, r *http.Request, svc *inbox.Service, raw, name string, br *bufio.Reader, headLen int) InboxUploadResult { + head := make([]byte, headLen) + if _, err := io.ReadFull(br, head); err != nil { + return s.inboxUploadRefused(r, name, 0, inbox.UploadErrInternal, "reading the file failed") + } + user := requestUser(r) + opts := inbox.TranscribeOptions{Actor: user, Audit: func(a inbox.TranscribeAudit) { + if s.audit != nil { + s.audit.Log(a.Actor, "inbox_transcribe", auditDetail(a.Fields()...), "") + } + }} + it, err := svc.AddMedia(ctx, inbox.MediaInput{Filename: raw, Head: head, Body: br, AddedBy: user}, opts) + if err != nil { + if isUploadTooLarge(err) { + return s.inboxUploadRefused(r, name, 0, inbox.UploadErrTooLarge, "the upload is too large") + } + var ue *inbox.UploadError + if errors.As(err, &ue) { + return s.inboxUploadRefused(r, name, 0, ue.Code, ue.Message) + } + if s.logger != nil { + s.logger.Warn("inbox: media upload failed", "error", err) + } + return s.inboxUploadRefused(r, name, 0, inbox.UploadErrInternal, "storing the file failed") + } + s.auditFromRequest(r, "inbox_upload", auditDetail("source", it.Source, "item", it.ID, "kind", it.UploadKind, "outcome", "added"), "") + return InboxUploadResult{Filename: name, OK: true, Source: it.Source, ItemID: it.ID, Title: it.Title, Kind: it.UploadKind} +} + // inboxUploadRefused audits a refused file and returns its result. The // audit records the reason code and size, not the file's content. func (s *Server) inboxUploadRefused(r *http.Request, name string, size int64, code, msg string) InboxUploadResult { diff --git a/src/pkg/dashboard/api_inbox_uploads_test.go b/src/pkg/dashboard/api_inbox_uploads_test.go index 18e7bc628e..06e487866b 100644 --- a/src/pkg/dashboard/api_inbox_uploads_test.go +++ b/src/pkg/dashboard/api_inbox_uploads_test.go @@ -277,3 +277,34 @@ func TestInboxUploadURL(t *testing.T) { t.Fatal("fetch_failed must be 502") } } + +func TestInboxUploadAudioStreamsAndPlays(t *testing.T) { + s, svc := newUploadTestServer(t, config.InboxUploads{}) + audio := "ID3\x04\x00\x00\x00\x00\x00\x00" + strings.Repeat("a", 4096) + rec := postUpload(s, "read-write", false, []uploadFile{{"standup.mp3", audio}}, nil) + if rec.Code != http.StatusCreated { + t.Fatalf("audio = %d %s", rec.Code, rec.Body.String()) + } + res := decodeUploadResponse(t, rec).Results + if len(res) != 1 || res[0].Kind != inbox.UploadKindAudio { + t.Fatalf("results = %+v", res) + } + // The generic same-origin stream serves it from disk. + req := httptest.NewRequest(http.MethodGet, "/api/inbox/upload/"+res[0].ItemID+"/audio/stream", nil) + req.Header.Set("X-Hive-Role", "read-write") + play := httptest.NewRecorder() + s.mux.ServeHTTP(play, req) + if play.Code != http.StatusOK || play.Body.Len() != len(audio) || !strings.HasPrefix(play.Header().Get("Content-Type"), "audio/mpeg") { + t.Fatalf("stream = %d %q %d", play.Code, play.Header().Get("Content-Type"), play.Body.Len()) + } + if rec := postUpload(s, "owner", true, []uploadFile{{"fake.mp3", "just text"}}, nil); rec.Code != http.StatusUnsupportedMediaType { + t.Fatalf("fake mp3 = %d", rec.Code) + } + items, _ := svc.Items(t.Context()) + if len(items) != 1 || !items[0].Pending { + t.Fatalf("items = %+v", items) + } + if st := svc.Settings().Uploads; st.MaxMediaBytes != config.DefaultInboxRecordingMaxBytes { + t.Fatalf("settings = %+v", st) + } +} diff --git a/src/pkg/dashboard/inbox_uploads.test.js b/src/pkg/dashboard/inbox_uploads.test.js index c22b61110c..3b5aa597d4 100644 --- a/src/pkg/dashboard/inbox_uploads.test.js +++ b/src/pkg/dashboard/inbox_uploads.test.js @@ -290,3 +290,12 @@ test('links: the Add field and a pasted URL are fetched by the hive, errors stay assert.match(api.inboxAddPanelHtml(), /id="inbox-add-url"[^>]*data-enter-action="inboxAddUrlFromField"/); assert.match(api.inboxUploadRowHtml({ id: 9, name: 'e.example', link: 'https://e.example/', status: 'uploading' }), /Fetching/); }); + +test('audio and video use the recording cap and are accepted by extension', () => { + const { api } = harness(); + const limits = Object.assign({}, UPLOADS, { max_files_per_drop: 10, media_extensions: ['.m4a', '.mp3', '.mp4'], max_media_bytes: 200 * MB }); + const rows = api.inboxUploadPlan([file('memo.m4a', 120 * MB), file('talk.mp4', 201 * MB), file('notes.md', 26 * MB), file('song.flac', 3)], limits); + assert.deepStrictEqual(rows.map(r => r.code || r.status), ['queued', 'too_large', 'too_large', 'unsupported_type']); + assert.match(rows[1].error, /the limit is 200\.0 MB/); + assert.match(rows[2].error, /the limit is 25\.0 MB/); +}); diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 0aa6c434a0..0e83789ef4 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -132,6 +132,7 @@ const meetingLabel = { zoom: 'Zoom', teams: 'Teams', 'google-meet': 'Google Meet', webex: 'Webex', 'slack-huddles': 'Slack', 'fireflies': 'Fireflies', 'fathom': 'Fathom', 'limitless': 'Limitless', 'granola': 'Granola', 'otter': 'Otter' }[it.source]; if (meetingLabel) return '<div class="inbox-pending muted-xs" title="The platform reports no transcript or summary for this recording yet">Not transcribed yet: ' + meetingLabel + ' has no transcript or summary for this recording (cloud transcription may still be processing or is disabled); the hive re-checks it on every sync.' + tx + '</div>'; if (it.source === 'hive-recording') return '<div class="inbox-pending muted-xs" title="Recorded in this hive\'s dashboard; no transcript yet">Not transcribed yet: recorded in this hive.' + (it.transcribe_unavailable === 'disabled' ? ' Transcription is off on this hive, so it is transcribed once an owner enables it in Inbox ⚙ → Transcription.' : tx) + '</div>'; + if (it.source === 'upload') return '<div class="inbox-pending muted-xs" title="Added to the Inbox as audio; it has no text until it is transcribed">Not transcribed yet: this audio was added to the Inbox without text.' + tx + '</div>'; if (it.source === 'discord') return '<div class="inbox-pending muted-xs" title="Discord has no transcripts; none was posted with this recording">Not transcribed yet: Discord has no built-in transcription and no transcript file was posted with this recording; play it here, or post a .vtt/.txt transcript in the same message. The hive re-checks it on every sync.' + tx + '</div>'; return '<div class="inbox-pending muted-xs" title="The MCP reports no transcript or summary block for this recording">Not transcribed yet in Plaud: Plaud has not transcribed this recording (open it in the Plaud app and run transcription); the hive re-checks it on every sync.' + tx + '</div>'; } @@ -145,7 +146,7 @@ if (!c) return '<div class="inbox-transcript muted-xs">Loading transcript…</div>'; if (c.error) return '<div class="inbox-transcript kc-error muted-xs">' + escapeHtml(c.error) + '</div>'; // A dropped document's text is the markdown the hive extracted. - if (it.source === 'upload' && !c.transcript) return '<div class="inbox-transcript"><div class="muted-xs"><strong>Extracted text</strong></div><pre class="inbox-transcript-text">' + escapeHtml(c.markdown || '') + '</pre></div>'; + if (it.source === 'upload' && !inboxItemHasAudio(it) && !c.transcript) return '<div class="inbox-transcript"><div class="muted-xs"><strong>Extracted text</strong></div><pre class="inbox-transcript-text">' + escapeHtml(c.markdown || '') + '</pre></div>'; let html = '<div class="inbox-transcript">'; if (c.transcript && c.transcript_label) html += '<div class="muted-xs"><strong>' + escapeHtml(c.transcript_label) + '</strong></div>'; if (c.transcript) html += '<pre class="inbox-transcript-text">' + escapeHtml(c.transcript) + '</pre>'; @@ -1247,13 +1248,17 @@ // decides from the content. function inboxUploadPlan(files, limits) { const maxFiles = Number(limits.max_files_per_drop) || 0; - const maxBytes = Number(limits.max_file_bytes) || 0; - const exts = Array.isArray(limits.extensions) ? limits.extensions : []; + const maxDoc = Number(limits.max_file_bytes) || 0; + // Audio and video use the recording cap (max_media_bytes). + const media = Array.isArray(limits.media_extensions) ? limits.media_extensions : []; + const maxMedia = Number(limits.max_media_bytes) || maxDoc; + const exts = (Array.isArray(limits.extensions) ? limits.extensions : []).concat(media); return Array.from(files || []).map((f, i) => { const name = (f && f.name) || 'pasted file'; const row = { name, size: Number(f && f.size) || 0, file: f, status: 'queued', progress: 0 }; const dot = name.lastIndexOf('.'); const ext = dot > 0 ? name.slice(dot).toLowerCase() : ''; + const maxBytes = media.indexOf(ext) >= 0 ? maxMedia : maxDoc; if (maxFiles && i >= maxFiles) { row.status = 'error'; row.code = 'too_many_files'; row.error = 'Too many files in one drop: at most ' + maxFiles + ' at a time.'; } else if (!row.size) { row.status = 'error'; row.code = 'empty'; row.error = 'The file is empty.'; } else if (maxBytes && row.size > maxBytes) { row.status = 'error'; row.code = 'too_large'; row.error = 'Too large: ' + inboxFmtBytes(row.size) + ' (the limit is ' + inboxFmtBytes(maxBytes) + ').'; } @@ -1283,8 +1288,8 @@ function inboxAddPanelHtml() { if (!_inboxAddOpen) return ''; const u = inboxUploadSettings(); - const exts = Array.isArray(u.extensions) ? u.extensions : []; - const limit = (u.max_files_per_drop ? 'up to ' + u.max_files_per_drop + ' files' : 'several files') + (u.max_file_bytes ? ' of ' + inboxFmtBytes(u.max_file_bytes) + ' each' : ''); + const exts = (Array.isArray(u.extensions) ? u.extensions : []).concat(Array.isArray(u.media_extensions) ? u.media_extensions : []); + const limit = (u.max_files_per_drop ? 'up to ' + u.max_files_per_drop + ' files' : 'several files') + (u.max_file_bytes ? ' of ' + inboxFmtBytes(u.max_file_bytes) + ' each' : '') + (u.max_media_bytes ? ' (audio and video up to ' + inboxFmtBytes(u.max_media_bytes) + ', transcribed when transcription is on)' : ''); return '<div class="inbox-add" id="inbox-add" role="group" aria-label="Add files to the Inbox">' + '<p class="muted-xs m-0">Drop files anywhere on the Inbox, paste them with Ctrl/Cmd+V, or choose them here: ' + escapeHtml(limit) + '. Supported: ' + escapeHtml(exts.join(' ')) + '. Files stay on this hive; their text reaches an LLM only when you share the item.</p>' + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxChooseFiles">Choose files…</button>' + diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index bc69e42cc6..0688fcb017 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -47867,6 +47867,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const meetingLabel = { zoom: 'Zoom', teams: 'Teams', 'google-meet': 'Google Meet', webex: 'Webex', 'slack-huddles': 'Slack', 'fireflies': 'Fireflies', 'fathom': 'Fathom', 'limitless': 'Limitless', 'granola': 'Granola', 'otter': 'Otter' }[it.source]; if (meetingLabel) return '<div class="inbox-pending muted-xs" title="The platform reports no transcript or summary for this recording yet">Not transcribed yet: ' + meetingLabel + ' has no transcript or summary for this recording (cloud transcription may still be processing or is disabled); the hive re-checks it on every sync.' + tx + '</div>'; if (it.source === 'hive-recording') return '<div class="inbox-pending muted-xs" title="Recorded in this hive\'s dashboard; no transcript yet">Not transcribed yet: recorded in this hive.' + (it.transcribe_unavailable === 'disabled' ? ' Transcription is off on this hive, so it is transcribed once an owner enables it in Inbox ⚙ → Transcription.' : tx) + '</div>'; + if (it.source === 'upload') return '<div class="inbox-pending muted-xs" title="Added to the Inbox as audio; it has no text until it is transcribed">Not transcribed yet: this audio was added to the Inbox without text.' + tx + '</div>'; if (it.source === 'discord') return '<div class="inbox-pending muted-xs" title="Discord has no transcripts; none was posted with this recording">Not transcribed yet: Discord has no built-in transcription and no transcript file was posted with this recording; play it here, or post a .vtt/.txt transcript in the same message. The hive re-checks it on every sync.' + tx + '</div>'; return '<div class="inbox-pending muted-xs" title="The MCP reports no transcript or summary block for this recording">Not transcribed yet in Plaud: Plaud has not transcribed this recording (open it in the Plaud app and run transcription); the hive re-checks it on every sync.' + tx + '</div>'; } @@ -47880,7 +47881,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> if (!c) return '<div class="inbox-transcript muted-xs">Loading transcript…</div>'; if (c.error) return '<div class="inbox-transcript kc-error muted-xs">' + escapeHtml(c.error) + '</div>'; // A dropped document's text is the markdown the hive extracted. - if (it.source === 'upload' && !c.transcript) return '<div class="inbox-transcript"><div class="muted-xs"><strong>Extracted text</strong></div><pre class="inbox-transcript-text">' + escapeHtml(c.markdown || '') + '</pre></div>'; + if (it.source === 'upload' && !inboxItemHasAudio(it) && !c.transcript) return '<div class="inbox-transcript"><div class="muted-xs"><strong>Extracted text</strong></div><pre class="inbox-transcript-text">' + escapeHtml(c.markdown || '') + '</pre></div>'; let html = '<div class="inbox-transcript">'; if (c.transcript && c.transcript_label) html += '<div class="muted-xs"><strong>' + escapeHtml(c.transcript_label) + '</strong></div>'; if (c.transcript) html += '<pre class="inbox-transcript-text">' + escapeHtml(c.transcript) + '</pre>'; @@ -48982,13 +48983,17 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> // decides from the content. function inboxUploadPlan(files, limits) { const maxFiles = Number(limits.max_files_per_drop) || 0; - const maxBytes = Number(limits.max_file_bytes) || 0; - const exts = Array.isArray(limits.extensions) ? limits.extensions : []; + const maxDoc = Number(limits.max_file_bytes) || 0; + // Audio and video use the recording cap (max_media_bytes). + const media = Array.isArray(limits.media_extensions) ? limits.media_extensions : []; + const maxMedia = Number(limits.max_media_bytes) || maxDoc; + const exts = (Array.isArray(limits.extensions) ? limits.extensions : []).concat(media); return Array.from(files || []).map((f, i) => { const name = (f && f.name) || 'pasted file'; const row = { name, size: Number(f && f.size) || 0, file: f, status: 'queued', progress: 0 }; const dot = name.lastIndexOf('.'); const ext = dot > 0 ? name.slice(dot).toLowerCase() : ''; + const maxBytes = media.indexOf(ext) >= 0 ? maxMedia : maxDoc; if (maxFiles && i >= maxFiles) { row.status = 'error'; row.code = 'too_many_files'; row.error = 'Too many files in one drop: at most ' + maxFiles + ' at a time.'; } else if (!row.size) { row.status = 'error'; row.code = 'empty'; row.error = 'The file is empty.'; } else if (maxBytes && row.size > maxBytes) { row.status = 'error'; row.code = 'too_large'; row.error = 'Too large: ' + inboxFmtBytes(row.size) + ' (the limit is ' + inboxFmtBytes(maxBytes) + ').'; } @@ -49018,8 +49023,8 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> function inboxAddPanelHtml() { if (!_inboxAddOpen) return ''; const u = inboxUploadSettings(); - const exts = Array.isArray(u.extensions) ? u.extensions : []; - const limit = (u.max_files_per_drop ? 'up to ' + u.max_files_per_drop + ' files' : 'several files') + (u.max_file_bytes ? ' of ' + inboxFmtBytes(u.max_file_bytes) + ' each' : ''); + const exts = (Array.isArray(u.extensions) ? u.extensions : []).concat(Array.isArray(u.media_extensions) ? u.media_extensions : []); + const limit = (u.max_files_per_drop ? 'up to ' + u.max_files_per_drop + ' files' : 'several files') + (u.max_file_bytes ? ' of ' + inboxFmtBytes(u.max_file_bytes) + ' each' : '') + (u.max_media_bytes ? ' (audio and video up to ' + inboxFmtBytes(u.max_media_bytes) + ', transcribed when transcription is on)' : ''); return '<div class="inbox-add" id="inbox-add" role="group" aria-label="Add files to the Inbox">' + '<p class="muted-xs m-0">Drop files anywhere on the Inbox, paste them with Ctrl/Cmd+V, or choose them here: ' + escapeHtml(limit) + '. Supported: ' + escapeHtml(exts.join(' ')) + '. Files stay on this hive; their text reaches an LLM only when you share the item.</p>' + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxChooseFiles">Choose files…</button>' + diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go index 82a59303e6..acc3c3fe99 100644 --- a/src/pkg/inbox/service.go +++ b/src/pkg/inbox/service.go @@ -148,6 +148,9 @@ func (s *Service) EnableUploads(u *UploadSource) { if u != nil && u.Limits == nil { u.Limits = func() config.InboxUploads { return s.Config().Uploads } } + if u != nil && u.RecordingLimits == nil { + u.RecordingLimits = func() config.InboxRecording { return s.Config().Recording } + } s.mu.Lock() defer s.mu.Unlock() s.uploads = u @@ -176,6 +179,27 @@ func (s *Service) AddUpload(ctx context.Context, in UploadInput) (Item, error) { return u.item(m), nil } +// AddMedia stores one dropped audio or video file and, when transcription +// is enabled and consented, queues its machine transcription at once +// (opts audits the run like a manual Transcribe). +func (s *Service) AddMedia(ctx context.Context, in MediaInput, opts TranscribeOptions) (Item, error) { + u := s.Uploads() + if u == nil { + return Item{}, ErrUploadsDisabled + } + m, err := u.AddMedia(ctx, in) + if err != nil { + return Item{}, err + } + it := u.item(m) + if s.Config().Transcription.Active() && s.transcriptStore() != nil { + if _, err := s.Transcribe(ctx, it, opts); err != nil { + s.Logger.Warn("inbox: upload transcription not queued", "item", it.Key(), "error", err) + } + } + return it, nil +} + // AddLink fetches one URL and returns its Inbox item. func (s *Service) AddLink(ctx context.Context, in LinkInput) (Item, error) { u := s.Uploads() diff --git a/src/pkg/inbox/upload_media.go b/src/pkg/inbox/upload_media.go new file mode 100644 index 0000000000..634c81867d --- /dev/null +++ b/src/pkg/inbox/upload_media.go @@ -0,0 +1,280 @@ +package inbox + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// Audio and video drops. The original is streamed to the upload store (it +// is never held in memory), capped by the recording limits +// (inbox.recording.max_bytes), played back from this hive's disk through the +// same-origin stream, and transcribed like a dashboard recording. Its +// original is deleted after the recordings retention (5 days by default); +// the transcript, the item and its fact stay. Video keeps its audio track +// for transcription only: there is no video playback. +const ( + // MediaSniffBytes is how much of a file the media sniffer reads. + MediaSniffBytes = 64 + // uploadMediaPendingNote is the fact's note while it has no transcript. + uploadMediaPendingNote = "_Added to the Inbox as audio; no transcript yet._" +) + +// mediaFormat is one accepted audio or video container. +type mediaFormat struct { + kind string + mime string +} + +// uploadMediaExtensions maps the accepted media extensions to their kind +// and the Content-Type playback uses. The content must match one of the +// signatures in sniffMedia as well. +var uploadMediaExtensions = map[string]mediaFormat{ + ".mp3": {UploadKindAudio, "audio/mpeg"}, + ".m4a": {UploadKindAudio, "audio/mp4"}, + ".aac": {UploadKindAudio, "audio/aac"}, + ".wav": {UploadKindAudio, "audio/wav"}, + ".ogg": {UploadKindAudio, "audio/ogg"}, + ".opus": {UploadKindAudio, "audio/ogg"}, + ".webm": {UploadKindAudio, "audio/webm"}, + ".mp4": {UploadKindVideo, "audio/mp4"}, + ".mov": {UploadKindVideo, "audio/mp4"}, +} + +// UploadMediaExtensions lists the accepted media extensions. +func UploadMediaExtensions() []string { + out := make([]string, 0, len(uploadMediaExtensions)) + for ext := range uploadMediaExtensions { + out = append(out, ext) + } + sort.Strings(out) + return out +} + +// sniffMedia names the container family of head ("" when it is not one of +// the accepted audio/video signatures). +func sniffMedia(head []byte) string { + switch { + case bytes.HasPrefix(head, []byte("ID3")): + return "mp3" + case len(head) >= 2 && head[0] == 0xff && head[1]&0xf6 == 0xf0: + return "aac" // ADTS + case len(head) >= 2 && head[0] == 0xff && head[1]&0xe0 == 0xe0: + return "mp3" // MPEG audio frame sync + case len(head) >= 12 && bytes.Equal(head[:4], []byte("RIFF")) && bytes.Equal(head[8:12], []byte("WAVE")): + return "wav" + case bytes.HasPrefix(head, []byte("OggS")): + return "ogg" + case bytes.HasPrefix(head, webmMagic): + return "webm" + case len(head) >= 8 && bytes.Equal(head[4:8], []byte("ftyp")): + return "mp4" + } + return "" +} + +// mediaFamilies are the signatures each extension may carry. +var mediaFamilies = map[string][]string{ + ".mp3": {"mp3"}, + ".m4a": {"mp4"}, + ".aac": {"aac", "mp4"}, + ".wav": {"wav"}, + ".ogg": {"ogg"}, + ".opus": {"ogg"}, + ".webm": {"webm"}, + ".mp4": {"mp4"}, + ".mov": {"mp4"}, +} + +// SniffMedia reports whether a dropped file is accepted audio or video, and +// as what. ok is false for anything else (the text pipeline decides then). +// A media extension whose content does not match is refused. +func SniffMedia(name string, head []byte) (mediaFormat, bool, *UploadError) { + ext := strings.ToLower(filepath.Ext(name)) + family := sniffMedia(head) + f, known := uploadMediaExtensions[ext] + if !known { + if family != "" && ext == "" { + return mediaFormat{}, false, uploadErr(UploadErrUnsupported, "audio or video files need their extension (.mp3, .m4a, .wav, .ogg, .webm, .mp4, ...)") + } + return mediaFormat{}, false, nil + } + for _, want := range mediaFamilies[ext] { + if family == want { + return f, true, nil + } + } + return mediaFormat{}, false, uploadErr(UploadErrUnsupported, "the file is named %s but its content is not %s audio or video", ext, strings.TrimPrefix(ext, ".")) +} + +// MediaInput is one dropped audio or video file, streamed. +type MediaInput struct { + Filename string + // Head is the first bytes already read for sniffing; Body is the rest. + Head []byte + Body io.Reader + AddedBy string +} + +func (u *UploadSource) recordingLimits() config.InboxRecording { + if u.RecordingLimits == nil { + return config.InboxRecording{} + } + return u.RecordingLimits() +} + +// AddMedia streams an audio or video file into the store under the +// recording size cap and the uploads quota, and writes a pending fact that +// the machine transcript later fills. +func (u *UploadSource) AddMedia(ctx context.Context, in MediaInput) (UploadMeta, error) { + name := CleanUploadName(in.Filename) + f, ok, uerr := SniffMedia(name, in.Head) + if uerr != nil { + return UploadMeta{}, uerr + } + if !ok { + return UploadMeta{}, uploadErr(UploadErrUnsupported, "not an audio or video file") + } + limit := u.recordingLimits().EffectiveMaxBytes() + id, err := u.Store.NewID() + if err != nil { + return UploadMeta{}, err + } + m := UploadMeta{ + ID: id, + Kind: f.kind, + MIME: f.mime, + Origin: UploadOriginFile, + Filename: name, + Title: firstNonEmpty(strings.TrimSuffix(name, filepath.Ext(name)), name), + AddedBy: firstNonEmpty(in.AddedBy, "local"), + AddedAt: u.now(), + } + h := sha256.New() + src := io.TeeReader(io.MultiReader(bytes.NewReader(in.Head), in.Body), h) + size, err := u.Store.PutStream(m, &ctxReader{ctx: ctx, r: src}, limit, u.limits().EffectiveQuotaBytes()) + if err != nil { + var ue *UploadError + if errors.As(err, &ue) { + return UploadMeta{}, ue + } + return UploadMeta{}, fmt.Errorf("storing the upload: %w", err) + } + m.Size = size + m.SHA256 = hex.EncodeToString(h.Sum(nil)) + if _, err := u.Store.UpdateMeta(m.ID, func(mm *UploadMeta) { mm.Size, mm.SHA256 = m.Size, m.SHA256 }); err != nil { + return UploadMeta{}, err + } + if u.ConnectorsDir == "" { + return m, nil + } + slug, layer, err := u.writeMediaFact(m) + if err != nil { + u.logger().Warn("inbox: upload fact not written", "upload", m.ID, "error", err) + return m, nil + } + return u.Store.UpdateMeta(m.ID, func(mm *UploadMeta) { mm.FactSlug, mm.FactLayer = slug, layer }) +} + +// ctxReader stops a stream once its context ends. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +// writeMediaFact writes the pending fact of an audio or video upload. +func (u *UploadSource) writeMediaFact(m UploadMeta) (string, string, error) { + u.factMu.Lock() + defer u.factMu.Unlock() + layer := u.layer() + dir := filepath.Join(u.ConnectorsDir, layer) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", "", err + } + slug := uploadFactPrefix + m.ID + sv := knowledge.SanitizeFrontmatterValue + var b strings.Builder + b.WriteString("---\n") + b.WriteString("title: " + sv(m.Title) + "\n") + b.WriteString("type: " + string(knowledge.FactReference) + "\n") + b.WriteString("layer: " + sv(layer) + "\n") + b.WriteString("status: " + connector.StatusActive + "\n") + b.WriteString("tags: [" + knowledge.SanitizeFrontmatterList([]string{UploadType, m.Origin, m.Kind}) + "]\n") + b.WriteString("source: " + UploadType + "\n") + b.WriteString("connector: " + UploadType + "\n") + b.WriteString("source_id: " + sv(m.ID) + "\n") + b.WriteString("attr_added_by: " + sv(m.AddedBy) + "\n") + b.WriteString("attr_recorded_at: " + m.AddedAt.UTC().Format(time.RFC3339) + "\n") + b.WriteString("attr_upload_filename: " + sv(m.Filename) + "\n") + b.WriteString("attr_upload_kind: " + sv(m.Kind) + "\n") + b.WriteString(uploadFactContentKey + ": " + connector.FactContentPending + "\n") + b.WriteString("synthesized: " + m.AddedAt.UTC().Format(time.RFC3339) + "\n") + b.WriteString("synced_at: " + u.now().Format(time.RFC3339) + "\n") + b.WriteString("---\n\n") + b.WriteString("# " + m.Title + "\n\n") + b.WriteString("- **Added:** " + m.AddedAt.UTC().Format(uploadFactDateLayout) + " by " + m.AddedBy + "\n") + b.WriteString("- **From:** " + UploadOrigin(m) + " (" + m.Kind + ", " + humanBytes(m.Size) + ")\n\n") + b.WriteString(uploadMediaPendingNote + "\n") + if err := writeFactAtomic(filepath.Join(dir, slug+".md"), b.String()); err != nil { + return "", "", err + } + if u.FactWritten != nil { + u.FactWritten(dir) + } + return slug, layer, nil +} + +// uploadAudioPathPrefix is the same-origin stream of an upload's audio. +const uploadAudioPathPrefix = "/api/inbox/" + UploadType + "/" + +// AudioLink implements AudioSource: the hive's own same-origin stream (the +// generic stream handler serves the file from disk through OpenAudio). +func (u *UploadSource) AudioLink(_ context.Context, it Item) (AudioLink, error) { + m, err := u.Store.Get(it.ID) + if err != nil || !uploadKindIsMedia(m.Kind) { + return AudioLink{}, ErrNoAudio + } + if m.OriginalDeletedAt != nil { + return AudioLink{}, fmt.Errorf("%w: the audio was deleted after the retention period", ErrNoAudio) + } + return AudioLink{URL: uploadAudioPathPrefix + m.ID + "/audio/stream", ExpiresAt: u.now().Add(recordingAudioLinkTTL)}, nil +} + +// OpenAudio implements AudioOpener: playback and transcription read the +// original from this hive's disk. +func (u *UploadSource) OpenAudio(_ context.Context, it Item) (LocalAudio, error) { + m, err := u.Store.Get(it.ID) + if err != nil || !uploadKindIsMedia(m.Kind) { + return LocalAudio{}, ErrNoAudio + } + f, err := u.Store.OpenOriginal(it.ID) + if err != nil { + return LocalAudio{}, fmt.Errorf("%w: the audio was deleted after the retention period", ErrNoAudio) + } + st, err := f.Stat() + if err != nil { + _ = f.Close() + return LocalAudio{}, err + } + return LocalAudio{File: f, Name: m.Filename, MIME: firstNonEmpty(m.MIME, "application/octet-stream"), Size: st.Size(), ModTime: st.ModTime()}, nil +} diff --git a/src/pkg/inbox/upload_media_test.go b/src/pkg/inbox/upload_media_test.go new file mode 100644 index 0000000000..632d333d2c --- /dev/null +++ b/src/pkg/inbox/upload_media_test.go @@ -0,0 +1,194 @@ +package inbox + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" + "github.com/hivecommons/hive/pkg/transcribe" +) + +var ( + mp3Head = []byte("ID3\x04\x00\x00\x00\x00\x00\x00audio") + m4aHead = []byte("\x00\x00\x00\x20ftypM4A \x00\x00\x00\x00") + wavHead = []byte("RIFF\x24\x00\x00\x00WAVEfmt ") + oggHead = []byte("OggS\x00\x02\x00\x00") + webmHead = []byte{0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86, 0x81} + adtsHead = []byte{0xff, 0xf1, 0x50, 0x80, 0x02, 0x1f, 0xfc} + syncHead = []byte{0xff, 0xfb, 0x90, 0x64, 0x00} +) + +func TestSniffMedia(t *testing.T) { + ok := []struct { + name string + head []byte + kind string + }{ + {"a.mp3", mp3Head, UploadKindAudio}, {"a.mp3", syncHead, UploadKindAudio}, {"a.m4a", m4aHead, UploadKindAudio}, + {"a.aac", adtsHead, UploadKindAudio}, {"a.wav", wavHead, UploadKindAudio}, {"a.ogg", oggHead, UploadKindAudio}, + {"a.opus", oggHead, UploadKindAudio}, {"a.webm", webmHead, UploadKindAudio}, {"a.mp4", m4aHead, UploadKindVideo}, + {"a.MOV", m4aHead, UploadKindVideo}, + } + for _, tc := range ok { + f, media, err := SniffMedia(tc.name, tc.head) + if err != nil || !media || f.kind != tc.kind { + t.Errorf("SniffMedia(%s) = %+v %v %v", tc.name, f, media, err) + } + } + // A media extension over the wrong content, and media without an + // extension, are refused; anything else is left to the text pipeline. + for name, head := range map[string][]byte{"a.mp3": []byte("plain text"), "a.wav": oggHead, "evil.mp4": []byte("MZ\x90\x00"), "clip": mp3Head} { + if _, media, err := SniffMedia(name, head); err == nil || media || err.Code != UploadErrUnsupported { + t.Errorf("SniffMedia(%s) = %v %v", name, media, err) + } + } + for name, head := range map[string][]byte{"notes.md": []byte("# hi"), "song.txt": mp3Head} { + if _, media, err := SniffMedia(name, head); err != nil || media { + t.Errorf("SniffMedia(%s) = %v %v, want left to the text pipeline", name, media, err) + } + } + if exts := UploadMediaExtensions(); len(exts) != len(uploadMediaExtensions) || exts[0] > exts[1] { + t.Fatalf("extensions = %v", exts) + } +} + +func addMedia(t *testing.T, e *uploadEnv, name string, head, body []byte) (Item, error) { + t.Helper() + return e.svc.AddMedia(context.Background(), MediaInput{Filename: name, Head: head, Body: bytes.NewReader(body), AddedBy: "ana"}, TranscribeOptions{Actor: "ana"}) +} + +func TestAddMediaStreamsToTheStoreAndPlaysLocally(t *testing.T) { + e := newUploadEnv(t, config.InboxConfig{}) + body := bytes.Repeat([]byte{0x55}, 4096) + it, err := addMedia(t, e, "standup.m4a", m4aHead, body) + if err != nil { + t.Fatal(err) + } + if it.UploadKind != UploadKindAudio || !it.Pending || it.FactState != connector.FactContentPending || it.Title != "standup" || it.Origin != "upload · standup.m4a" { + t.Fatalf("item = %+v", it) + } + m, err := e.src.Store.Get(it.ID) + if err != nil || m.Size != int64(len(m4aHead)+len(body)) || m.SHA256 == "" || m.MIME != "audio/mp4" { + t.Fatalf("meta = %+v, %v", m, err) + } + info, err := os.Stat(filepath.Join(e.src.Store.Dir, it.ID, uploadOriginalFile)) + if err != nil || info.Mode().Perm() != uploadFileMode { + t.Fatalf("original mode = %v", err) + } + fact, _ := os.ReadFile(it.FactPath) + if !strings.Contains(string(fact), "attr_upload_content: pending") || !strings.Contains(string(fact), uploadMediaPendingNote) { + t.Fatalf("fact = %s", fact) + } + // Playback and transcription read the file from this hive's disk. + a, local, err := e.svc.OpenLocalAudio(context.Background(), it) + if err != nil || !local || a.MIME != "audio/mp4" || a.Size != m.Size { + t.Fatalf("OpenLocalAudio = %+v %v %v", a, local, err) + } + got, _ := io.ReadAll(a.File) + _ = a.File.Close() + if !bytes.HasPrefix(got, m4aHead) || len(got) != int(m.Size) { + t.Fatal("audio bytes differ") + } + link, err := e.src.AudioLink(context.Background(), it) + if err != nil || link.URL != "/api/inbox/upload/"+it.ID+"/audio/stream" { + t.Fatalf("AudioLink = %+v, %v", link, err) + } + c, err := e.svc.Content(context.Background(), it) + if err != nil || !c.Pending || c.Markdown != "" { + t.Fatalf("content = %+v, %v", c, err) + } + // Documents have no audio. + doc, _ := e.svc.AddUpload(context.Background(), UploadInput{Filename: "n.md", Data: []byte("# n")}) + if _, err := e.src.AudioLink(context.Background(), doc); !errors.Is(err, ErrNoAudio) { + t.Fatalf("doc audio = %v", err) + } + if _, err := e.src.OpenAudio(context.Background(), doc); !errors.Is(err, ErrNoAudio) { + t.Fatalf("doc open = %v", err) + } +} + +func TestAddMediaCapsQuotaAndRetention(t *testing.T) { + e := newUploadEnv(t, config.InboxConfig{Recording: config.InboxRecording{MaxBytes: 1024}, Uploads: config.InboxUploads{QuotaBytes: 1 << 20}}) + code := func(err error) string { + var ue *UploadError + if errors.As(err, &ue) { + return ue.Code + } + return "?" + } + if _, err := addMedia(t, e, "big.mp3", mp3Head, bytes.Repeat([]byte{1}, 2048)); code(err) != UploadErrTooLarge { + t.Fatalf("over the recording cap = %v", err) + } + if list, _ := e.src.Store.List(); len(list) != 0 { + t.Fatal("a refused stream left an upload behind") + } + if _, err := addMedia(t, e, "fake.mp3", []byte("not audio"), nil); code(err) != UploadErrUnsupported { + t.Fatalf("mismatch = %v", err) + } + it, err := addMedia(t, e, "memo.mp3", mp3Head, []byte("x")) + if err != nil { + t.Fatal(err) + } + st := e.svc.Settings().Uploads + if st.MaxMediaBytes != 1024 || st.MediaRetentionDays != config.DefaultInboxRecordingRetentionDays || len(st.MediaExtensions) == 0 { + t.Fatalf("settings = %+v", st) + } + // The original goes after the recordings retention; the item stays. + e.now = e.now.Add(time.Duration(config.DefaultInboxRecordingRetentionDays+1) * hoursPerDay * time.Hour) + if n, err := e.src.SweepRetention(); err != nil || n != 1 { + t.Fatalf("retention = %d, %v", n, err) + } + if _, err := e.src.AudioLink(context.Background(), it); !errors.Is(err, ErrNoAudio) { + t.Fatalf("expired audio link = %v", err) + } + if _, err := e.src.OpenAudio(context.Background(), it); !errors.Is(err, ErrNoAudio) { + t.Fatalf("expired open = %v", err) + } + items, _ := e.svc.Items(context.Background()) + if len(items) != 1 { + t.Fatalf("items after retention = %d", len(items)) + } + // A full quota refuses before streaming. + full := newUploadEnv(t, config.InboxConfig{Uploads: config.InboxUploads{QuotaBytes: 1}}) + if _, err := full.svc.AddUpload(context.Background(), UploadInput{Filename: "a.txt", Data: []byte("a")}); code(err) != UploadErrQuota { + t.Fatalf("doc over quota = %v", err) + } + _ = os.MkdirAll(full.src.Store.Dir, 0o700) + _ = os.WriteFile(filepath.Join(full.src.Store.Dir, "pad"), []byte("pp"), 0o600) + if _, err := addMedia(t, full, "a.mp3", mp3Head, nil); code(err) != UploadErrQuota { + t.Fatalf("media over quota = %v", err) + } +} + +func TestAddMediaQueuesTranscription(t *testing.T) { + e := newUploadEnv(t, config.InboxConfig{Transcription: activeTx()}) + e.svc.Run = func(fn func()) { fn() } + e.svc.EnableTranscription(NewTranscriptStore(filepath.Join(t.TempDir(), "transcripts"))) + e.svc.NewTranscriber = func(config.InboxTranscription) (transcribe.Transcriber, error) { + return &transcribe.Fake{Result: transcribe.Result{Segments: []transcribe.Segment{{Text: "hello"}}, Duration: time.Second}}, nil + } + e.svc.Getenv = func(k string) string { + if k == "DEEPGRAM_API_KEY" { + return "k" + } + return "" + } + it, err := addMedia(t, e, "memo.ogg", oggHead, []byte("body")) + if err != nil { + t.Fatal(err) + } + items, _ := e.svc.Items(context.Background()) + for _, got := range items { + if got.ID == it.ID && (got.Transcription == nil || got.Transcription.Status == "") { + t.Fatalf("no transcription was queued: %+v", got) + } + } +} diff --git a/src/pkg/inbox/upload_source.go b/src/pkg/inbox/upload_source.go index f0954abde6..a14e3c433c 100644 --- a/src/pkg/inbox/upload_source.go +++ b/src/pkg/inbox/upload_source.go @@ -66,6 +66,10 @@ type UploadSource struct { FactWritten func(dir string) // Fetcher fetches links; nil refuses links. Fetcher *LinkFetcher + // RecordingLimits returns inbox.recording: audio and video drops use + // its size cap and, unless inbox.uploads.media_retention_days is set, + // its retention. nil uses the recording defaults. + RecordingLimits func() config.InboxRecording // PDF runs the hive-pdftext sandbox; nil reads HIVE_PDF_* on first use. PDF *PDFRuntime pdfMu sync.Mutex @@ -149,6 +153,18 @@ func (u *UploadSource) item(m UploadMeta) Item { it.FactPath = filepath.Join(u.ConnectorsDir, m.FactLayer, m.FactSlug+".md") it.FactState = connector.FactContentReady } + if uploadKindIsMedia(m.Kind) { + // Audio and video have no text of their own: pending until the + // machine transcript is written into the fact. + it.Pending = true + it.FactState = connector.FactContentPending + if it.FactPath != "" { + if data, err := os.ReadFile(it.FactPath); err == nil { + fm, _ := splitFrontMatter(string(data)) + it.FactState = factState(fm, true) + } + } + } return it } @@ -180,6 +196,9 @@ func (u *UploadSource) Fetch(_ context.Context, it Item) (Content, error) { if err != nil { return Content{}, err } + if uploadKindIsMedia(m.Kind) { + return Content{Title: firstNonEmpty(m.Title, m.Filename, m.ID), Pending: true, Provenance: UploadProvenance(m)}, nil + } return Content{ Title: firstNonEmpty(m.Title, m.Filename, m.ID), Markdown: strings.TrimSpace(text), @@ -429,6 +448,10 @@ type UploadSettings struct { MediaRetentionDays int `json:"media_retention_days"` DocumentRetentionDays int `json:"document_retention_days"` Extensions []string `json:"extensions"` + // MediaExtensions are the audio/video extensions, capped by + // MaxMediaBytes (inbox.recording.max_bytes) instead of MaxFileBytes. + MediaExtensions []string `json:"media_extensions"` + MaxMediaBytes int64 `json:"max_media_bytes"` // Links reports that URLs can be added (the hive has a link fetcher). Links bool `json:"links"` } @@ -443,7 +466,9 @@ func (u *UploadSource) Settings() UploadSettings { MaxFilesPerDrop: lim.EffectiveMaxFilesPerDrop(), QuotaBytes: lim.EffectiveQuotaBytes(), UsedBytes: used, - MediaRetentionDays: lim.EffectiveMediaRetentionDays(), + MediaRetentionDays: u.mediaRetentionDays(), + MediaExtensions: UploadMediaExtensions(), + MaxMediaBytes: u.recordingLimits().EffectiveMaxBytes(), DocumentRetentionDays: lim.EffectiveDocumentRetentionDays(), Extensions: UploadSupportedExtensions(), Links: u.Fetcher != nil, @@ -452,6 +477,15 @@ func (u *UploadSource) Settings() UploadSettings { // SweepRetention applies inbox.uploads retention to stored originals. func (u *UploadSource) SweepRetention() (int, error) { + return u.Store.SweepRetention(u.mediaRetentionDays(), u.limits().EffectiveDocumentRetentionDays()) +} + +// mediaRetentionDays is inbox.uploads.media_retention_days when set (or its +// environment override), else the recordings retention. +func (u *UploadSource) mediaRetentionDays() int { lim := u.limits() - return u.Store.SweepRetention(lim.EffectiveMediaRetentionDays(), lim.EffectiveDocumentRetentionDays()) + if lim.MediaRetentionDays > 0 || strings.TrimSpace(os.Getenv(config.EnvInboxUploadsMediaRetentionDays)) != "" { + return lim.EffectiveMediaRetentionDays() + } + return u.recordingLimits().EffectiveRetentionDays() } diff --git a/src/pkg/inbox/upload_store.go b/src/pkg/inbox/upload_store.go index 7ab5298374..9e5d7ef33d 100644 --- a/src/pkg/inbox/upload_store.go +++ b/src/pkg/inbox/upload_store.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -43,6 +44,8 @@ var ErrUploadNotFound = errors.New("unknown upload") type UploadMeta struct { ID string `json:"id"` Kind string `json:"kind"` + // MIME is the playback Content-Type of an audio or video upload. + MIME string `json:"mime,omitempty"` // Origin is "upload" for a dropped file, "link" for a fetched URL. Origin string `json:"origin"` Filename string `json:"filename,omitempty"` @@ -143,6 +146,81 @@ func (s *UploadStore) Put(meta UploadMeta, original []byte, extracted string, qu return nil } +// PutStream stores an upload whose original is streamed from r (audio and +// video), at most limit bytes and within quota. The extracted text starts +// empty (the machine transcript is kept by the transcript store). It +// returns the stored size. +func (s *UploadStore) PutStream(meta UploadMeta, r io.Reader, limit, quota int64) (int64, error) { + dir, err := s.dir(meta.ID) + if err != nil { + return 0, err + } + s.mu.Lock() + defer s.mu.Unlock() + if quota > 0 { + used, err := s.usage() + if err != nil { + return 0, err + } + if used >= quota { + return 0, uploadErr(UploadErrQuota, "the Inbox upload quota is full (%s of %s used): remove old uploads or raise inbox.uploads.quota_bytes", humanBytes(used), humanBytes(quota)) + } + if room := quota - used; limit <= 0 || room < limit { + limit = room + } + } + if err := os.MkdirAll(s.Dir, uploadDirMode); err != nil { + return 0, err + } + if err := os.Chmod(s.Dir, uploadDirMode); err != nil { + return 0, err + } + if err := os.Mkdir(dir, uploadDirMode); err != nil { + return 0, err + } + ok := false + defer func() { + if !ok { + _ = os.RemoveAll(dir) + } + }() + f, err := os.OpenFile(filepath.Join(dir, uploadOriginalFile), os.O_WRONLY|os.O_CREATE|os.O_EXCL, uploadFileMode) + if err != nil { + return 0, err + } + n, err := io.Copy(f, io.LimitReader(r, limit+1)) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return 0, err + } + if n > limit { + return 0, uploadErr(UploadErrTooLarge, "the file is too large or the upload quota is nearly full (the limit for this file is %s)", humanBytes(limit)) + } + if n == 0 { + return 0, uploadErr(UploadErrEmpty, "the file is empty") + } + if err := writeNew(filepath.Join(dir, uploadTextFile), nil); err != nil { + return 0, err + } + meta.Size = n + if err := s.writeMeta(dir, meta); err != nil { + return 0, err + } + ok = true + return n, nil +} + +// OpenOriginal opens an upload's original file (until retention removes it). +func (s *UploadStore) OpenOriginal(id string) (*os.File, error) { + dir, err := s.dir(id) + if err != nil { + return nil, err + } + return os.Open(filepath.Join(dir, uploadOriginalFile)) +} + // writeNew creates path exclusively with uploadFileMode. func writeNew(path string, data []byte) error { f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, uploadFileMode)