From b192799d718b20fb56102bdc1d5c29e1dc6daccf Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 08:12:11 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Inbox:=20auto-transcribe=20hive=20r?= =?UTF-8?q?ecordings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A finished recording is queued for transcription with the hive's provider as soon as it is finalized. AudioOpener (now in its own file, general to any source whose audio is on this hive's disk, such as uploads) lets the transcriber open the local file directly, so the spoke never calls its own HTTP API. Recording does not imply transcription consent: nothing runs until inbox.transcription is enabled with consent, after which recordings count as opted in regardless of mode and the sweep (also run from the recording ticker) catches up on recordings made while it was off. Signed-off-by: Andy Anderson --- .../added-inbox-recording-autotranscribe.md | 1 + src/cmd/hive/boot_inbox.go | 9 + src/docs/inbox-recording.md | 21 ++ src/pkg/dashboard/api_inbox_recordings.go | 4 + src/pkg/dashboard/inbox_recorder.test.js | 2 +- .../dashboard/static-src/js/inbox-recorder.js | 4 +- src/pkg/dashboard/static/index.html | 4 +- src/pkg/inbox/audioopener.go | 81 ++++++ src/pkg/inbox/recordings_edges_test.go | 244 ++++++++++++++++++ src/pkg/inbox/recordings_transcribe_test.go | 167 ++++++++++++ src/pkg/inbox/recordingsource.go | 85 +++--- src/pkg/inbox/transcription.go | 12 +- src/pkg/inbox/transcription_sweep.go | 10 +- 13 files changed, 603 insertions(+), 41 deletions(-) create mode 100644 changelog.d/added-inbox-recording-autotranscribe.md create mode 100644 src/pkg/inbox/audioopener.go create mode 100644 src/pkg/inbox/recordings_edges_test.go create mode 100644 src/pkg/inbox/recordings_transcribe_test.go diff --git a/changelog.d/added-inbox-recording-autotranscribe.md b/changelog.d/added-inbox-recording-autotranscribe.md new file mode 100644 index 0000000000..ffe6abca46 --- /dev/null +++ b/changelog.d/added-inbox-recording-autotranscribe.md @@ -0,0 +1 @@ +- Inbox recordings are transcribed automatically when you stop, with the hive's transcription provider (auto: the built-in whisper first), with the usual progress bar and time-left estimate. The transcriber reads the recording from the hive's own disk (new `AudioOpener`; no HTTP call to itself). Nothing runs until `inbox.transcription` is enabled with consent; from then on every dashboard recording counts as opted in whatever the mode, and recordings made while it was off are picked up by the transcription sweep. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index 463a5829e5..2ef3a03936 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -273,11 +273,20 @@ func (b *boot) startInboxRecordingSweep(svc *inbox.Service) { if b.dashSrv != nil { b.dashSrv.AuditLog(a.Actor, a.Action, a.Detail(), "") } + }, TranscribeAudit: func(a inbox.TranscribeAudit) { + if b.dashSrv != nil { + b.dashSrv.AuditLog(a.Actor, "inbox_transcribe", a.Detail(), "") + } }} sweep := func() { if _, err := svc.RecordingSweep(opts); err != nil { b.logger.Warn("inbox recording sweep", "error", err) } + // Recordings finished while transcription was off are transcribed + // once it is enabled with consent. + tctx, cancel := context.WithTimeout(ctx, inboxSweepTimeout) + b.inboxTranscribeSweep(tctx, svc) + cancel() } go func() { sweep() diff --git a/src/docs/inbox-recording.md b/src/docs/inbox-recording.md index 3b68942d08..6f3ba2802e 100644 --- a/src/docs/inbox-recording.md +++ b/src/docs/inbox-recording.md @@ -34,6 +34,27 @@ you. Good practice: notice; - do not record people who have not been told. +## Transcription + +When you stop, the hive transcribes the recording automatically with its +[transcription provider](inbox-transcription.md) (`provider: auto` picks the +built-in whisper first, so the audio stays on the hive), with the same +progress bar and time-left estimate as any other Inbox transcription. The +transcriber reads the file from the hive's own disk; nothing is fetched over +HTTP. The transcript is written into the recording's Knowledge fact. + +- Recording does **not** replace the hive-wide transcription consent: nothing + is transcribed until an owner enables `inbox.transcription` with + `consent_acknowledged: true`. Once that is done, every dashboard recording + counts as opted in, whatever `inbox.transcription.mode` says. +- While transcription is off you can still record; the consent notice says + the recording will not be transcribed yet, and the item shows why. The + transcription sweep (every 10 minutes and after connector syncs) picks up + such recordings once transcription is enabled, as long as their audio has + not expired. +- Transcription limits (`max_bytes`, `max_duration_minutes`, daily and + monthly budgets) apply as for every other source. + ## Retention | What | Kept | diff --git a/src/pkg/dashboard/api_inbox_recordings.go b/src/pkg/dashboard/api_inbox_recordings.go index f1b67e4636..402acf7636 100644 --- a/src/pkg/dashboard/api_inbox_recordings.go +++ b/src/pkg/dashboard/api_inbox_recordings.go @@ -76,6 +76,10 @@ func (s *Server) inboxRecordingOptions(r *http.Request) inbox.RecordingOptions { if s.audit != nil { s.audit.Log(a.Actor, a.Action, auditDetail(a.Fields...), "") } + }, TranscribeAudit: func(a inbox.TranscribeAudit) { + if s.audit != nil { + s.audit.Log(a.Actor, "inbox_transcribe", auditDetail(a.Fields()...), "") + } }} } diff --git a/src/pkg/dashboard/inbox_recorder.test.js b/src/pkg/dashboard/inbox_recorder.test.js index feaf6b1ed2..d2188ba79a 100644 --- a/src/pkg/dashboard/inbox_recorder.test.js +++ b/src/pkg/dashboard/inbox_recorder.test.js @@ -142,7 +142,7 @@ test('consent text: responsibility, retention, transcription off hint, tab parti const tab = lib.inboxRecConsentText({ tab: true, retentionDays: 1, transcribes: true }).join(' '); assert.match(tab, /remote participants of that meeting too: they must have been told/); assert.match(tab, /1 day and/); - assert.match(tab, /can be transcribed on this hive/); + assert.match(tab, /transcribes the recording automatically/); }); test('upload queue resumes after a network failure and keeps order', async () => { diff --git a/src/pkg/dashboard/static-src/js/inbox-recorder.js b/src/pkg/dashboard/static-src/js/inbox-recorder.js index 6e7c842a9d..94e10c8efd 100644 --- a/src/pkg/dashboard/static-src/js/inbox-recorder.js +++ b/src/pkg/dashboard/static-src/js/inbox-recorder.js @@ -109,7 +109,7 @@ ]; if (o.tab) parts.push('Including tab audio captures the remote participants of that meeting too: they must have been told that you are recording.'); parts.push('The raw audio is kept on this hive for ' + days + ' day' + (days === 1 ? '' : 's') + ' and then deleted; its transcript and Knowledge fact are kept.'); - parts.push(o.transcribes ? 'When you stop, it can be transcribed on this hive with 🎧 Transcribe.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); + parts.push(o.transcribes ? 'When you stop, this hive transcribes the recording automatically.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); return parts; } const INBOX_REC_CONSENT_ACK = 'I have told everyone who will be recorded, and I am responsible for this recording.'; @@ -531,7 +531,7 @@ if (m.uploader) html += '
' + escapeHtml(inboxRecUploadText(m.uploader.state)) + '
'; if (m.warning) html += ''; if (m.error) html += ''; - if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'It appears in the Inbox below; use 🎧 Transcribe on it.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; + if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'The hive is transcribing it now; follow the progress on the item in the Inbox below.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; html += '
'; if (m.state === 'recording') html += ''; if (m.state === 'paused') html += ''; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 4844e744cd..bc69e42cc6 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -49276,7 +49276,7 @@ ]; if (o.tab) parts.push('Including tab audio captures the remote participants of that meeting too: they must have been told that you are recording.'); parts.push('The raw audio is kept on this hive for ' + days + ' day' + (days === 1 ? '' : 's') + ' and then deleted; its transcript and Knowledge fact are kept.'); - parts.push(o.transcribes ? 'When you stop, it can be transcribed on this hive with 🎧 Transcribe.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); + parts.push(o.transcribes ? 'When you stop, this hive transcribes the recording automatically.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); return parts; } const INBOX_REC_CONSENT_ACK = 'I have told everyone who will be recorded, and I am responsible for this recording.'; @@ -49698,7 +49698,7 @@ if (m.uploader) html += '
' + escapeHtml(inboxRecUploadText(m.uploader.state)) + '
'; if (m.warning) html += ''; if (m.error) html += ''; - if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'It appears in the Inbox below; use 🎧 Transcribe on it.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; + if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'The hive is transcribing it now; follow the progress on the item in the Inbox below.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; html += '
'; if (m.state === 'recording') html += ''; if (m.state === 'paused') html += ''; diff --git a/src/pkg/inbox/audioopener.go b/src/pkg/inbox/audioopener.go new file mode 100644 index 0000000000..cb38e46dea --- /dev/null +++ b/src/pkg/inbox/audioopener.go @@ -0,0 +1,81 @@ +package inbox + +import ( + "context" + "io" + "os" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// LocalAudio is an audio (or video) file on this hive's own disk. +type LocalAudio struct { + File *os.File + // Name is the file name http.ServeContent uses for type fallbacks. + Name string + // MIME is the media type it is served and transcribed as. + MIME string + Size int64 + ModTime time.Time +} + +// AudioOpener is optionally implemented by any source whose audio lives on +// this hive (dashboard recordings, uploaded audio and video files, ...). +// Playback serves the file directly with Range support, and transcription +// reads it from disk: nothing makes an HTTP call to the hive itself. A +// source that implements AudioOpener should also implement AudioSource so +// the item counts as having audio. OpenAudio returns ErrNoAudio (or an +// error wrapping it) when the item has no audio file. +type AudioOpener interface { + OpenAudio(ctx context.Context, it Item) (LocalAudio, error) +} + +// localTranscribeAudio opens a hive-local file for a transcription run. The +// first open happens here so the provider gets the file's MIME type and +// size; retries reopen it. The source checked the container when the file +// arrived; decoding is the provider's job (the sandboxed whisper subprocess +// by default). The returned cleanup closes the first handle if the provider +// never read it. +func (s *Service) localTranscribeAudio(ctx context.Context, it Item, op AudioOpener, cfg config.InboxTranscription, rec *TranscriptRecord, job *txJob) (transcribe.Audio, func(), error) { + s.txProgress(job, transcribe.Progress{Phase: transcribe.PhaseDownloading, ProcessedSeconds: -1}) + first, err := op.OpenAudio(ctx, it) + if err != nil { + return transcribe.Audio{}, nil, err + } + maxBytes := cfg.EffectiveMaxBytes() + rec.AudioSize = first.Size + pending := &first + cleanup := func() { + if pending != nil { + _ = pending.File.Close() + pending = nil + } + } + if maxBytes > 0 && first.Size > maxBytes { + cleanup() + return transcribe.Audio{}, nil, transcribe.ErrTooLarge + } + a := transcribe.Audio{ + MIME: first.MIME, + Size: first.Size, + Duration: time.Duration(it.DurationMS) * time.Millisecond, + Lang: cfg.EffectiveLanguage(), + Progress: func(p transcribe.Progress) { s.txProgress(job, p) }, + } + a.Open = func(ctx context.Context) (io.ReadCloser, error) { + la := pending + pending = nil + if la == nil { + s.txProgress(job, transcribe.Progress{Phase: transcribe.PhaseDownloading, ProcessedSeconds: -1}) + reopened, err := op.OpenAudio(ctx, it) + if err != nil { + return nil, err + } + la = &reopened + } + return readCloser{Reader: transcribe.LimitReader(la.File, maxBytes), Closer: la.File}, nil + } + return a, cleanup, nil +} diff --git a/src/pkg/inbox/recordings_edges_test.go b/src/pkg/inbox/recordings_edges_test.go new file mode 100644 index 0000000000..b1d7fdb249 --- /dev/null +++ b/src/pkg/inbox/recordings_edges_test.go @@ -0,0 +1,244 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +func TestRecordingErrorsAndDisabledPaths(t *testing.T) { + if !strings.Contains((&RecordingActiveError{ID: "x"}).Error(), "(x)") { + t.Fatal("active error lacks the id") + } + if !strings.Contains((&RecordingGapError{Got: 3, Expected: 1}).Error(), "seq=1") { + t.Fatal("gap error lacks the expected seq") + } + if d := (RecordingAudit{Fields: []string{"a", "1", "b", "2", "odd"}}).Detail(); d != "a=1, b=2" { + t.Fatalf("detail = %q", d) + } + + // No recording source at all. + bare := NewService(config.InboxConfig{}, NewStore(filepath.Join(t.TempDir(), "s.json")), nil, nil, nil) + if _, err := bare.StartRecording(RecordingStart{}, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("start without source = %v", err) + } + if _, err := bare.AppendRecordingChunk("x", "u", 0, []byte{1}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("append without source = %v", err) + } + if _, err := bare.FinalizeRecording("x", "u", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("finalize without source = %v", err) + } + if _, err := bare.AbortRecording("x", "u", false, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("abort without source = %v", err) + } + if _, err := bare.OpenRecordingAudio("x"); !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("open without source = %v", err) + } + if res, err := bare.RecordingSweep(RecordingOptions{}); err != nil || len(res.Expired)+len(res.Finalized)+len(res.Removed) != 0 { + t.Fatalf("sweep without source = %+v, %v", res, err) + } + if bare.RecordingSettings().Enabled { + t.Fatal("enabled without a source") + } + if _, local, err := bare.OpenLocalAudio(context.Background(), Item{Source: "plaud"}); local || err != nil { + t.Fatalf("non-local source = %v, %v", local, err) + } + + svc, src, _ := newRecordingService(t, enabledRecording()) + for _, call := range []func() error{ + func() error { _, err := svc.AppendRecordingChunk("missing", "ana", 0, []byte{1}); return err }, + func() error { _, err := svc.FinalizeRecording("missing", "ana", 0, RecordingOptions{}); return err }, + func() error { _, err := svc.OpenRecordingAudio("missing"); return err }, + } { + if err := call(); !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("unknown recording = %v", err) + } + } + if _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: RecordingConsentVersion}, RecordingOptions{}); !errors.Is(err, ErrRecordingBadRequest) { + t.Fatalf("start without user = %v", err) + } + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.OpenRecordingAudio(m.ID); !errors.Is(err, ErrNoAudio) { + t.Fatalf("audio of an active recording = %v", err) + } + if _, err := src.AudioLink(context.Background(), Item{ID: m.ID}); !errors.Is(err, ErrNoAudio) { + t.Fatalf("link of an active recording = %v", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(2)); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingClosed) { + t.Fatalf("finalize twice = %v", err) + } + if _, err := svc.AbortRecording(m.ID, "ana", false, RecordingOptions{}); !errors.Is(err, ErrRecordingClosed) { + t.Fatalf("abort finalized = %v", err) + } + if got := len(src.Store.List()); got != 1 { + t.Fatalf("list = %d", got) + } + // A recording without a fact directory still finalizes. + if err := (&RecordingSource{}).writeFact(RecordingMeta{}); err != nil { + t.Fatalf("writeFact without dir = %v", err) + } +} + +func TestRecordingStoreLoadSkipsJunk(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"not-an-id", "20261116t090000-0000000a", "20261116t090000-0000000b"} { + if err := os.MkdirAll(filepath.Join(dir, name), recordingDirMode); err != nil { + t.Fatal(err) + } + } + _ = os.WriteFile(filepath.Join(dir, "20261116t090000-0000000a", recordingMetaFile), []byte("{not json"), recordingFileMode) + _ = os.WriteFile(filepath.Join(dir, "20261116t090000-0000000b", recordingMetaFile), []byte(`{"id":"other"}`), recordingFileMode) + _ = os.WriteFile(filepath.Join(dir, "stray.txt"), []byte("x"), recordingFileMode) + if got := NewRecordingStore(dir).List(); len(got) != 0 { + t.Fatalf("junk loaded: %+v", got) + } + if got := NewRecordingStore(filepath.Join(dir, "absent")).UsedBytes(); got != 0 { + t.Fatalf("absent dir used = %d", got) + } +} + +func TestMP4BoxWalkEdges(t *testing.T) { + large := make([]byte, mp4LargeBoxHeader+4) + binary.BigEndian.PutUint32(large[:4], 1) + copy(large[4:8], "mdat") + binary.BigEndian.PutUint64(large[8:16], uint64(len(large))) + stream := append(mp4Box("ftyp", []byte("iso5")), large...) + if types, err := MP4TopLevelBoxes(bytes.NewReader(stream)); err != nil || strings.Join(types, ",") != "ftyp,mdat" { + t.Fatalf("64-bit box = %v, %v", types, err) + } + toEnd := append(mp4Box("ftyp", nil), 0, 0, 0, 0, 'm', 'd', 'a', 't', 9, 9) + if types, err := MP4TopLevelBoxes(bytes.NewReader(toEnd)); err != nil || len(types) != 2 { + t.Fatalf("size-0 box = %v, %v", types, err) + } + for name, bad := range map[string][]byte{ + "tiny size": {0, 0, 0, 4, 'f', 't', 'y', 'p'}, + "trailing bytes": append(mp4Box("ftyp", nil), 1, 2, 3), + "bad large size": {0, 0, 0, 1, 'm', 'd', 'a', 't', 0, 0, 0, 0, 0, 0, 0, 2}, + "short large": {0, 0, 0, 1, 'm', 'd', 'a', 't', 0, 0}, + } { + if _, err := MP4TopLevelBoxes(bytes.NewReader(bad)); err == nil { + t.Errorf("%s accepted", name) + } + } + dir := t.TempDir() + webm := filepath.Join(dir, "a.webm") + _ = os.WriteFile(webm, []byte("nope"), recordingFileMode) + if w := checkRecordingContainer(webm, RecordingContainerWebM); w == "" { + t.Fatal("bad webm header passed") + } + mp4 := filepath.Join(dir, "a.mp4") + _ = os.WriteFile(mp4, mp4Box("moov", nil), recordingFileMode) + if w := checkRecordingContainer(mp4, RecordingContainerMP4); w == "" { + t.Fatal("mp4 without ftyp passed") + } + if w := checkRecordingContainer(filepath.Join(dir, "absent"), RecordingContainerMP4); w == "" { + t.Fatal("missing file passed") + } +} + +// reopenOpener counts opens so the retry path is covered. +type reopenOpener struct { + path string + opens int +} + +func (r *reopenOpener) OpenAudio(context.Context, Item) (LocalAudio, error) { + r.opens++ + f, err := os.Open(r.path) + if err != nil { + return LocalAudio{}, err + } + st, _ := f.Stat() + return LocalAudio{File: f, MIME: "audio/webm", Size: st.Size()}, nil +} + +func TestLocalTranscribeAudioReopensForRetries(t *testing.T) { + path := filepath.Join(t.TempDir(), "a.webm") + if err := os.WriteFile(path, webmChunk(12), recordingFileMode); err != nil { + t.Fatal(err) + } + svc, _, _ := newRecordingService(t, enabledRecording()) + op := &reopenOpener{path: path} + rec := &TranscriptRecord{} + job := svc.txEnqueue(Item{Source: "x", ID: "y"}, "fake", "m", transcribe.Speed{}) + defer svc.txDequeue(job) + a, cleanup, err := svc.localTranscribeAudio(context.Background(), Item{ID: "y"}, op, config.InboxTranscription{}, rec, job) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if a.MIME != "audio/webm" || a.Size != 16 || rec.AudioSize != 16 { + t.Fatalf("audio = %+v, rec size %d", a, rec.AudioSize) + } + for i := 0; i < 2; i++ { + rc, err := a.Open(context.Background()) + if err != nil { + t.Fatal(err) + } + n, _ := io.Copy(io.Discard, rc) + _ = rc.Close() + if n != 16 { + t.Fatalf("read %d bytes", n) + } + } + if op.opens != 2 { + t.Fatalf("opens = %d, want the first handle reused then one reopen", op.opens) + } + // An unused first handle is closed by cleanup; an open error is returned. + _, cleanup2, err := svc.localTranscribeAudio(context.Background(), Item{ID: "y"}, op, config.InboxTranscription{}, rec, job) + if err != nil { + t.Fatal(err) + } + cleanup2() + cleanup2() + if _, _, err := svc.localTranscribeAudio(context.Background(), Item{}, &reopenOpener{path: filepath.Join(t.TempDir(), "absent")}, config.InboxTranscription{}, rec, job); err == nil { + t.Fatal("missing file opened") + } + if _, _, err := svc.localTranscribeAudio(context.Background(), Item{}, op, config.InboxTranscription{MaxBytes: 4}, rec, job); !errors.Is(err, transcribe.ErrTooLarge) { + t.Fatalf("over max_bytes = %v", err) + } +} + +func TestRecordingAutoTranscribeSweepActorAndBusy(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + var actors []string + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(8)); err != nil { + t.Fatal(err) + } + clock.t = clock.t.Add(3 * time.Hour) + if _, err := svc.RecordingSweep(RecordingOptions{TranscribeAudit: func(a TranscribeAudit) { actors = append(actors, a.Actor) }}); err != nil { + t.Fatal(err) + } + svc.Wait() + if fake.Calls() != 1 || len(actors) != 1 || actors[0] != RecordingSweepActor { + t.Fatalf("idle-finalized recording: calls %d, actors %v", fake.Calls(), actors) + } + // Already transcribed: finalize-time auto-transcribe is a no-op. + got, _ := src.Store.Get(m.ID) + svc.autoTranscribeRecording(src, got, RecordingOptions{}) + svc.Wait() + if fake.Calls() != 1 { + t.Fatal("a transcribed recording was transcribed again") + } + // No actor at all falls back to the auto-transcribe actor. + got.ID = "unknown" + svc.autoTranscribeRecording(src, got, RecordingOptions{}) +} diff --git a/src/pkg/inbox/recordings_transcribe_test.go b/src/pkg/inbox/recordings_transcribe_test.go new file mode 100644 index 0000000000..064805c555 --- /dev/null +++ b/src/pkg/inbox/recordings_transcribe_test.go @@ -0,0 +1,167 @@ +package inbox + +import ( + "context" + "errors" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// noSelfHTTP fails a test that makes any HTTP call: a recording's audio is +// read from disk, never fetched from the hive's own endpoint. +type noSelfHTTP struct{ t *testing.T } + +func (n noSelfHTTP) RoundTrip(r *http.Request) (*http.Response, error) { + n.t.Errorf("unexpected HTTP request for a hive recording: %s", r.URL) + return nil, errors.New("no HTTP") +} + +func withTranscription(t *testing.T, svc *Service, tx config.InboxTranscription) *transcribe.Fake { + t.Helper() + cfg := svc.Config() + cfg.Transcription = tx + svc.UpdateConfig(cfg) + fake := &transcribe.Fake{Result: transcribe.Result{Segments: []transcribe.Segment{{Text: "decisions were made"}}, Duration: 2 * time.Second}} + svc.NewTranscriber = func(config.InboxTranscription) (transcribe.Transcriber, error) { return fake, nil } + svc.Getenv = func(string) string { return "" } + svc.AudioClient = &http.Client{Transport: noSelfHTTP{t}} + svc.TranscribeBackoff = func(int) time.Duration { return 0 } + return fake +} + +func recordAndFinalize(t *testing.T, svc *Service, user string, opts RecordingOptions) RecordingMeta { + t.Helper() + m := startRecording(t, svc, user, "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, user, 0, webmChunk(30)); err != nil { + t.Fatal(err) + } + if _, err := svc.AppendRecordingChunk(m.ID, user, 1, []byte("cluster")); err != nil { + t.Fatal(err) + } + fin, err := svc.FinalizeRecording(m.ID, user, 2000, opts) + if err != nil { + t.Fatal(err) + } + return fin +} + +func TestRecordingAutoTranscribedOnFinalize(t *testing.T) { + svc, src, _ := newRecordingService(t, enabledRecording()) + // Manual mode: a recording is transcribed anyway, once transcription + // is enabled with consent. + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + var audits []TranscribeAudit + m := recordAndFinalize(t, svc, "ana", RecordingOptions{Actor: "ana", TranscribeAudit: func(a TranscribeAudit) { audits = append(audits, a) }}) + svc.Wait() + if fake.Calls() != 1 { + t.Fatalf("transcriber calls = %d, want 1 (auto-transcribe on finalize)", fake.Calls()) + } + if fake.Bytes() != m.Bytes { + t.Fatalf("transcriber read %d bytes, want the %d-byte local file", fake.Bytes(), m.Bytes) + } + rec, ok := svc.transcriptStore().Get(RecordingType, m.ID) + if !ok || rec.Status != TranscriptionDone || !strings.Contains(rec.Text, "decisions were made") || rec.AudioSize != m.Bytes { + t.Fatalf("transcript = %+v, %v", rec, ok) + } + if len(audits) != 1 || audits[0].Actor != "ana" || audits[0].Outcome != TranscriptionDone { + t.Fatalf("transcribe audits = %+v", audits) + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].Pending || items[0].FactState != connector.FactContentMachine { + t.Fatalf("item after transcription = %+v", items) + } + fact, _ := os.ReadFile(items[0].FactPath) + if !strings.Contains(string(fact), "decisions were made") { + t.Fatalf("fact lacks the transcript:\n%s", fact) + } + c, err := svc.Content(context.Background(), items[0]) + if err != nil || !strings.Contains(c.Transcript, "decisions were made") { + t.Fatalf("content = %+v, %v", c, err) + } + _ = src +} + +func TestRecordingNotTranscribedUntilTranscriptionEnabled(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{}) + if svc.RecordingSettings().Transcribes { + t.Fatal("settings say recordings are transcribed while transcription is off") + } + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + svc.Wait() + if fake.Calls() != 0 { + t.Fatal("a recording was transcribed while transcription is disabled") + } + if _, ok := svc.transcriptStore().Get(RecordingType, m.ID); ok { + t.Fatal("a transcript record exists while transcription is disabled") + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].TranscribeUnavailable != TranscribeBlockedDisabled { + t.Fatalf("item = %+v, want transcribe_unavailable=disabled", items) + } + // Enabled without consent is not enough (and is rejected by config). + if n, _ := svc.TranscribeSweep(context.Background(), TranscribeOptions{}); n != 0 { + t.Fatalf("sweep without transcription queued %d", n) + } + // Enabled with consent, still manual mode: the sweep picks it up. + withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + n, err := svc.TranscribeSweep(context.Background(), TranscribeOptions{Actor: AutoTranscribeActor}) + svc.Wait() + if err != nil || n != 1 { + t.Fatalf("sweep queued %d, %v; want the recording", n, err) + } + if rec, ok := svc.transcriptStore().Get(RecordingType, m.ID); !ok || rec.Status != TranscriptionDone { + t.Fatalf("transcript after enabling = %+v", rec) + } + if !svc.RecordingSettings().Transcribes { + t.Fatal("settings do not report automatic transcription") + } +} + +func TestRecordingExpiredAudioIsNotTranscribed(t *testing.T) { + svc, _, clock := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{}) + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + clock.t = clock.t.Add(config.DefaultInboxRecordingRetentionDays * 24 * time.Hour) + if res, _ := svc.RecordingSweep(RecordingOptions{}); len(res.Expired) != 1 { + t.Fatalf("not expired: %+v", res) + } + withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + if n, _ := svc.TranscribeSweep(context.Background(), TranscribeOptions{}); n != 0 { + t.Fatalf("sweep queued %d expired recordings", n) + } + svc.Wait() + if fake.Calls() != 0 { + t.Fatal("expired audio was sent to the transcriber") + } + it, ok, _ := svc.Find(context.Background(), RecordingType, m.ID) + if !ok { + t.Fatal("expired recording not listed") + } + if _, err := svc.Transcribe(context.Background(), it, TranscribeOptions{}); err != nil { + t.Fatalf("manual transcribe of expired audio = %v (want a skipped state)", err) + } + rec, _ := svc.transcriptStore().Get(RecordingType, m.ID) + if rec.Status != TranscriptionSkipped || !strings.HasPrefix(rec.Reason, SkipUnsupported) { + t.Fatalf("expired manual transcribe = %+v, want skipped unsupported", rec) + } +} + +func TestRecordingTranscriptionTooLarge(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true, MaxBytes: 10}) + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + svc.Wait() + rec, _ := svc.transcriptStore().Get(RecordingType, m.ID) + if rec.Status != TranscriptionSkipped || !strings.HasPrefix(rec.Reason, SkipTooLarge) { + t.Fatalf("over max_bytes = %+v", rec) + } + _ = fake +} diff --git a/src/pkg/inbox/recordingsource.go b/src/pkg/inbox/recordingsource.go index 09cfca495b..654fb1d81d 100644 --- a/src/pkg/inbox/recordingsource.go +++ b/src/pkg/inbox/recordingsource.go @@ -27,22 +27,6 @@ const ( RecordingSweepActor = "inbox-recording-sweep" ) -// LocalAudio is an audio file on this hive's own disk. -type LocalAudio struct { - File *os.File - Name string - MIME string - Size int64 - ModTime time.Time -} - -// AudioOpener is optionally implemented by sources whose audio lives on this -// hive: playback serves the file directly, and nothing makes an HTTP call to -// the hive itself. -type AudioOpener interface { - OpenAudio(ctx context.Context, it Item) (LocalAudio, error) -} - // RecordingSource lists the recordings made in this hive's dashboard. It is // a builtin source (listed whatever inbox.sources says). Each finalized // recording has a Knowledge fact (personal layer) that starts pending and @@ -81,28 +65,33 @@ func (r *RecordingSource) List(ctx context.Context) ([]Item, error) { if m.State != RecordingFinalized { continue } - it := Item{ - Source: RecordingType, Connector: RecordingConnector, ID: m.ID, Title: m.Title, - RecordedAt: m.CreatedAt, DurationMS: m.DurationMS, Tags: []string{"recording"}, - RecordedBy: m.RecordedBy, AudioExpired: m.AudioExpired(), AudioRetentionDays: days, - Pending: true, FactState: connector.FactContentPending, - } - if m.IncludesTabAudio { - it.Tags = append(it.Tags, "tab-audio") - } - path := r.factPath(m.ID) - if data, err := os.ReadFile(path); err == nil { - fm, body := splitFrontMatter(string(data)) - fact := itemFromFact(fm, body, RecordingLayer, path) - it.FactSlug, it.FactPath, it.Layer, it.FactState = fact.FactSlug, fact.FactPath, fact.Layer, fact.FactState - it.Summary, it.HasTranscript, it.Pending, it.Excerpt = fact.Summary, fact.HasTranscript, fact.Pending, fact.Excerpt - } - out = append(out, it) + out = append(out, r.item(m, days)) } sortItems(out) return out, nil } +// item is the Inbox item of one finalized recording, with its fact. +func (r *RecordingSource) item(m RecordingMeta, retentionDays int) Item { + it := Item{ + Source: RecordingType, Connector: RecordingConnector, ID: m.ID, Title: m.Title, + RecordedAt: m.CreatedAt, DurationMS: m.DurationMS, Tags: []string{"recording"}, + RecordedBy: m.RecordedBy, AudioExpired: m.AudioExpired(), AudioRetentionDays: retentionDays, + Pending: true, FactState: connector.FactContentPending, + } + if m.IncludesTabAudio { + it.Tags = append(it.Tags, "tab-audio") + } + path := r.factPath(m.ID) + if data, err := os.ReadFile(path); err == nil { + fm, body := splitFrontMatter(string(data)) + fact := itemFromFact(fm, body, RecordingLayer, path) + it.FactSlug, it.FactPath, it.Layer, it.FactState = fact.FactSlug, fact.FactPath, fact.Layer, fact.FactState + it.Summary, it.HasTranscript, it.Pending, it.Excerpt = fact.Summary, fact.HasTranscript, fact.Pending, fact.Excerpt + } + return it +} + // Fetch implements Source from the recording's fact. func (r *RecordingSource) Fetch(ctx context.Context, it Item) (Content, error) { provenance := RecordingType + "://recordings/" + it.ID @@ -228,6 +217,9 @@ func (a RecordingAudit) Detail() string { type RecordingOptions struct { Actor string Audit func(RecordingAudit) + // TranscribeAudit receives the inbox_transcribe entry of the automatic + // transcription a finished recording starts. + TranscribeAudit func(TranscribeAudit) } func (o RecordingOptions) audit(action string, fields ...string) { @@ -328,6 +320,33 @@ func (s *Service) recordingFinalized(src *RecordingSource, m RecordingMeta, opts } opts.audit("inbox_recording_finalize", "recording", m.ID, "chunks", strconv.Itoa(m.Chunks), "bytes", strconv.FormatInt(m.Bytes, 10), "duration_ms", strconv.FormatInt(m.DurationMS, 10), "auto", strconv.FormatBool(m.AutoFinalized)) + s.autoTranscribeRecording(src, m, opts) +} + +// autoTranscribeRecording queues the transcription of a finished recording +// with the hive's provider (auto: whisper first). Recording does not stand in +// for the hive-wide transcription consent: nothing runs until +// inbox.transcription is enabled with consent_acknowledged, and then every +// recording counts as opted in whatever inbox.transcription.mode says (the +// transcription sweep picks up recordings finished while it was off). +func (s *Service) autoTranscribeRecording(src *RecordingSource, m RecordingMeta, opts RecordingOptions) { + if !s.Config().Transcription.Active() || m.AudioExpired() { + return + } + it := src.item(m, s.Config().Recording.EffectiveRetentionDays()) + if !it.Pending { + return + } + actor := opts.Actor + if actor == "" { + actor = AutoTranscribeActor + } + st, err := s.Transcribe(context.Background(), it, TranscribeOptions{Actor: actor, Audit: opts.TranscribeAudit}) + if err != nil { + s.Logger.Warn("inbox: auto-transcribe recording", "recording", m.ID, "error", err) + return + } + s.Logger.Info("inbox: recording queued for transcription", "recording", m.ID, "status", st.Status) } // AbortRecording discards an in-progress recording. diff --git a/src/pkg/inbox/transcription.go b/src/pkg/inbox/transcription.go index a0a2781207..943ba8306a 100644 --- a/src/pkg/inbox/transcription.go +++ b/src/pkg/inbox/transcription.go @@ -412,7 +412,7 @@ func (s *Service) runTranscription(it Item, cfg config.InboxTranscription, rec T } case errors.Is(err, transcribe.ErrTooLarge): rec.Status, rec.Reason = TranscriptionSkipped, fmt.Sprintf("%s: the audio is larger than %d MB (inbox.transcription.max_bytes)", SkipTooLarge, cfg.EffectiveMaxBytes()/bytesPerMB) - case errors.Is(err, ErrNoAudio), errors.Is(err, connector.ErrPlaudNoAudio): + case errors.Is(err, ErrNoAudio), errors.Is(err, connector.ErrPlaudNoAudio), errors.Is(err, ErrRecordingExpired): rec.Status, rec.Reason = TranscriptionSkipped, SkipUnsupported+": "+err.Error() default: rec.Status, rec.Reason = TranscriptionFailed, err.Error() @@ -462,6 +462,16 @@ func (s *Service) transcribeItem(ctx context.Context, it Item, cfg config.InboxT if err != nil { return transcribe.Result{}, err } + if op, ok := s.source(it.Source).(AudioOpener); ok { + // Audio kept on this hive (recordings, uploads) is read from disk: + // the spoke never makes an HTTP call to itself. + audio, cleanup, err := s.localTranscribeAudio(ctx, it, op, cfg, rec, job) + if err != nil { + return transcribe.Result{}, err + } + defer cleanup() + return tr.Transcribe(ctx, audio) + } link, err := s.AudioLink(ctx, it) if err != nil { return transcribe.Result{}, err diff --git a/src/pkg/inbox/transcription_sweep.go b/src/pkg/inbox/transcription_sweep.go index 75880e2df7..9fa4f82ca7 100644 --- a/src/pkg/inbox/transcription_sweep.go +++ b/src/pkg/inbox/transcription_sweep.go @@ -32,7 +32,12 @@ const ( func (s *Service) TranscribeSweep(ctx context.Context, opts TranscribeOptions) (queued int, err error) { cfg := s.Config().Transcription store := s.transcriptStore() - if store == nil || !cfg.Active() || cfg.EffectiveMode() != config.InboxTranscriptionModeAuto { + // Dashboard recordings are transcribed whatever the mode: recording one + // opts it in, once transcription is enabled with consent. + if store == nil || !cfg.Active() { + return 0, nil + } + if cfg.EffectiveMode() != config.InboxTranscriptionModeAuto && s.recordingSource() == nil { return 0, nil } items, err := s.Items(ctx) @@ -45,7 +50,8 @@ func (s *Service) TranscribeSweep(ctx context.Context, opts TranscribeOptions) ( if itemsLeft <= 0 { break } - if seen[it.Key()] || !it.Transcribable || !(cfg.AutoSource(it.Source) || cfg.AutoSource(it.Connector)) || !autoDue(it.Transcription, now) { + optedIn := it.Source == RecordingType || cfg.AutoSource(it.Source) || cfg.AutoSource(it.Connector) + if seen[it.Key()] || !it.Transcribable || !optedIn || !autoDue(it.Transcription, now) { continue } seen[it.Key()] = true