diff --git a/changelog.d/added-inbox-recording-autotranscribe.md b/changelog.d/added-inbox-recording-autotranscribe.md new file mode 100644 index 0000000000..ffe6abca46 --- /dev/null +++ b/changelog.d/added-inbox-recording-autotranscribe.md @@ -0,0 +1 @@ +- Inbox recordings are transcribed automatically when you stop, with the hive's transcription provider (auto: the built-in whisper first), with the usual progress bar and time-left estimate. The transcriber reads the recording from the hive's own disk (new `AudioOpener`; no HTTP call to itself). Nothing runs until `inbox.transcription` is enabled with consent; from then on every dashboard recording counts as opted in whatever the mode, and recordings made while it was off are picked up by the transcription sweep. diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index 463a5829e5..2ef3a03936 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -273,11 +273,20 @@ func (b *boot) startInboxRecordingSweep(svc *inbox.Service) { if b.dashSrv != nil { b.dashSrv.AuditLog(a.Actor, a.Action, a.Detail(), "") } + }, TranscribeAudit: func(a inbox.TranscribeAudit) { + if b.dashSrv != nil { + b.dashSrv.AuditLog(a.Actor, "inbox_transcribe", a.Detail(), "") + } }} sweep := func() { if _, err := svc.RecordingSweep(opts); err != nil { b.logger.Warn("inbox recording sweep", "error", err) } + // Recordings finished while transcription was off are transcribed + // once it is enabled with consent. + tctx, cancel := context.WithTimeout(ctx, inboxSweepTimeout) + b.inboxTranscribeSweep(tctx, svc) + cancel() } go func() { sweep() diff --git a/src/docs/inbox-recording.md b/src/docs/inbox-recording.md index 3b68942d08..6f3ba2802e 100644 --- a/src/docs/inbox-recording.md +++ b/src/docs/inbox-recording.md @@ -34,6 +34,27 @@ you. Good practice: notice; - do not record people who have not been told. +## Transcription + +When you stop, the hive transcribes the recording automatically with its +[transcription provider](inbox-transcription.md) (`provider: auto` picks the +built-in whisper first, so the audio stays on the hive), with the same +progress bar and time-left estimate as any other Inbox transcription. The +transcriber reads the file from the hive's own disk; nothing is fetched over +HTTP. The transcript is written into the recording's Knowledge fact. + +- Recording does **not** replace the hive-wide transcription consent: nothing + is transcribed until an owner enables `inbox.transcription` with + `consent_acknowledged: true`. Once that is done, every dashboard recording + counts as opted in, whatever `inbox.transcription.mode` says. +- While transcription is off you can still record; the consent notice says + the recording will not be transcribed yet, and the item shows why. The + transcription sweep (every 10 minutes and after connector syncs) picks up + such recordings once transcription is enabled, as long as their audio has + not expired. +- Transcription limits (`max_bytes`, `max_duration_minutes`, daily and + monthly budgets) apply as for every other source. + ## Retention | What | Kept | diff --git a/src/pkg/dashboard/api_inbox_recordings.go b/src/pkg/dashboard/api_inbox_recordings.go index f1b67e4636..402acf7636 100644 --- a/src/pkg/dashboard/api_inbox_recordings.go +++ b/src/pkg/dashboard/api_inbox_recordings.go @@ -76,6 +76,10 @@ func (s *Server) inboxRecordingOptions(r *http.Request) inbox.RecordingOptions { if s.audit != nil { s.audit.Log(a.Actor, a.Action, auditDetail(a.Fields...), "") } + }, TranscribeAudit: func(a inbox.TranscribeAudit) { + if s.audit != nil { + s.audit.Log(a.Actor, "inbox_transcribe", auditDetail(a.Fields()...), "") + } }} } diff --git a/src/pkg/dashboard/inbox_recorder.test.js b/src/pkg/dashboard/inbox_recorder.test.js index feaf6b1ed2..d2188ba79a 100644 --- a/src/pkg/dashboard/inbox_recorder.test.js +++ b/src/pkg/dashboard/inbox_recorder.test.js @@ -142,7 +142,7 @@ test('consent text: responsibility, retention, transcription off hint, tab parti const tab = lib.inboxRecConsentText({ tab: true, retentionDays: 1, transcribes: true }).join(' '); assert.match(tab, /remote participants of that meeting too: they must have been told/); assert.match(tab, /1 day and/); - assert.match(tab, /can be transcribed on this hive/); + assert.match(tab, /transcribes the recording automatically/); }); test('upload queue resumes after a network failure and keeps order', async () => { diff --git a/src/pkg/dashboard/static-src/js/inbox-recorder.js b/src/pkg/dashboard/static-src/js/inbox-recorder.js index 6e7c842a9d..94e10c8efd 100644 --- a/src/pkg/dashboard/static-src/js/inbox-recorder.js +++ b/src/pkg/dashboard/static-src/js/inbox-recorder.js @@ -109,7 +109,7 @@ ]; if (o.tab) parts.push('Including tab audio captures the remote participants of that meeting too: they must have been told that you are recording.'); parts.push('The raw audio is kept on this hive for ' + days + ' day' + (days === 1 ? '' : 's') + ' and then deleted; its transcript and Knowledge fact are kept.'); - parts.push(o.transcribes ? 'When you stop, it can be transcribed on this hive with 🎧 Transcribe.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); + parts.push(o.transcribes ? 'When you stop, this hive transcribes the recording automatically.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); return parts; } const INBOX_REC_CONSENT_ACK = 'I have told everyone who will be recorded, and I am responsible for this recording.'; @@ -531,7 +531,7 @@ if (m.uploader) html += '
' + escapeHtml(inboxRecUploadText(m.uploader.state)) + '
'; if (m.warning) html += ''; if (m.error) html += ''; - if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'It appears in the Inbox below; use 🎧 Transcribe on it.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; + if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'The hive is transcribing it now; follow the progress on the item in the Inbox below.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; html += '
'; if (m.state === 'recording') html += ''; if (m.state === 'paused') html += ''; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 4844e744cd..bc69e42cc6 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -49276,7 +49276,7 @@ ]; if (o.tab) parts.push('Including tab audio captures the remote participants of that meeting too: they must have been told that you are recording.'); parts.push('The raw audio is kept on this hive for ' + days + ' day' + (days === 1 ? '' : 's') + ' and then deleted; its transcript and Knowledge fact are kept.'); - parts.push(o.transcribes ? 'When you stop, it can be transcribed on this hive with 🎧 Transcribe.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); + parts.push(o.transcribes ? 'When you stop, this hive transcribes the recording automatically.' : 'Transcription is off on this hive, so this recording is not transcribed until an owner enables it in Inbox ⚙ → Transcription.'); return parts; } const INBOX_REC_CONSENT_ACK = 'I have told everyone who will be recorded, and I am responsible for this recording.'; @@ -49698,7 +49698,7 @@ if (m.uploader) html += '
' + escapeHtml(inboxRecUploadText(m.uploader.state)) + '
'; if (m.warning) html += ''; if (m.error) html += ''; - if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'It appears in the Inbox below; use 🎧 Transcribe on it.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; + if (m.state === 'done') html += '
' + escapeHtml(m.transcribes ? 'The hive is transcribing it now; follow the progress on the item in the Inbox below.' : 'It appears in the Inbox below. Transcription is off on this hive, so it is not transcribed yet.') + '
'; html += '
'; if (m.state === 'recording') html += ''; if (m.state === 'paused') html += ''; diff --git a/src/pkg/inbox/audioopener.go b/src/pkg/inbox/audioopener.go new file mode 100644 index 0000000000..cb38e46dea --- /dev/null +++ b/src/pkg/inbox/audioopener.go @@ -0,0 +1,81 @@ +package inbox + +import ( + "context" + "io" + "os" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// LocalAudio is an audio (or video) file on this hive's own disk. +type LocalAudio struct { + File *os.File + // Name is the file name http.ServeContent uses for type fallbacks. + Name string + // MIME is the media type it is served and transcribed as. + MIME string + Size int64 + ModTime time.Time +} + +// AudioOpener is optionally implemented by any source whose audio lives on +// this hive (dashboard recordings, uploaded audio and video files, ...). +// Playback serves the file directly with Range support, and transcription +// reads it from disk: nothing makes an HTTP call to the hive itself. A +// source that implements AudioOpener should also implement AudioSource so +// the item counts as having audio. OpenAudio returns ErrNoAudio (or an +// error wrapping it) when the item has no audio file. +type AudioOpener interface { + OpenAudio(ctx context.Context, it Item) (LocalAudio, error) +} + +// localTranscribeAudio opens a hive-local file for a transcription run. The +// first open happens here so the provider gets the file's MIME type and +// size; retries reopen it. The source checked the container when the file +// arrived; decoding is the provider's job (the sandboxed whisper subprocess +// by default). The returned cleanup closes the first handle if the provider +// never read it. +func (s *Service) localTranscribeAudio(ctx context.Context, it Item, op AudioOpener, cfg config.InboxTranscription, rec *TranscriptRecord, job *txJob) (transcribe.Audio, func(), error) { + s.txProgress(job, transcribe.Progress{Phase: transcribe.PhaseDownloading, ProcessedSeconds: -1}) + first, err := op.OpenAudio(ctx, it) + if err != nil { + return transcribe.Audio{}, nil, err + } + maxBytes := cfg.EffectiveMaxBytes() + rec.AudioSize = first.Size + pending := &first + cleanup := func() { + if pending != nil { + _ = pending.File.Close() + pending = nil + } + } + if maxBytes > 0 && first.Size > maxBytes { + cleanup() + return transcribe.Audio{}, nil, transcribe.ErrTooLarge + } + a := transcribe.Audio{ + MIME: first.MIME, + Size: first.Size, + Duration: time.Duration(it.DurationMS) * time.Millisecond, + Lang: cfg.EffectiveLanguage(), + Progress: func(p transcribe.Progress) { s.txProgress(job, p) }, + } + a.Open = func(ctx context.Context) (io.ReadCloser, error) { + la := pending + pending = nil + if la == nil { + s.txProgress(job, transcribe.Progress{Phase: transcribe.PhaseDownloading, ProcessedSeconds: -1}) + reopened, err := op.OpenAudio(ctx, it) + if err != nil { + return nil, err + } + la = &reopened + } + return readCloser{Reader: transcribe.LimitReader(la.File, maxBytes), Closer: la.File}, nil + } + return a, cleanup, nil +} diff --git a/src/pkg/inbox/recordings_edges_test.go b/src/pkg/inbox/recordings_edges_test.go new file mode 100644 index 0000000000..b1d7fdb249 --- /dev/null +++ b/src/pkg/inbox/recordings_edges_test.go @@ -0,0 +1,244 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +func TestRecordingErrorsAndDisabledPaths(t *testing.T) { + if !strings.Contains((&RecordingActiveError{ID: "x"}).Error(), "(x)") { + t.Fatal("active error lacks the id") + } + if !strings.Contains((&RecordingGapError{Got: 3, Expected: 1}).Error(), "seq=1") { + t.Fatal("gap error lacks the expected seq") + } + if d := (RecordingAudit{Fields: []string{"a", "1", "b", "2", "odd"}}).Detail(); d != "a=1, b=2" { + t.Fatalf("detail = %q", d) + } + + // No recording source at all. + bare := NewService(config.InboxConfig{}, NewStore(filepath.Join(t.TempDir(), "s.json")), nil, nil, nil) + if _, err := bare.StartRecording(RecordingStart{}, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("start without source = %v", err) + } + if _, err := bare.AppendRecordingChunk("x", "u", 0, []byte{1}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("append without source = %v", err) + } + if _, err := bare.FinalizeRecording("x", "u", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("finalize without source = %v", err) + } + if _, err := bare.AbortRecording("x", "u", false, RecordingOptions{}); !errors.Is(err, ErrRecordingDisabled) { + t.Fatalf("abort without source = %v", err) + } + if _, err := bare.OpenRecordingAudio("x"); !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("open without source = %v", err) + } + if res, err := bare.RecordingSweep(RecordingOptions{}); err != nil || len(res.Expired)+len(res.Finalized)+len(res.Removed) != 0 { + t.Fatalf("sweep without source = %+v, %v", res, err) + } + if bare.RecordingSettings().Enabled { + t.Fatal("enabled without a source") + } + if _, local, err := bare.OpenLocalAudio(context.Background(), Item{Source: "plaud"}); local || err != nil { + t.Fatalf("non-local source = %v, %v", local, err) + } + + svc, src, _ := newRecordingService(t, enabledRecording()) + for _, call := range []func() error{ + func() error { _, err := svc.AppendRecordingChunk("missing", "ana", 0, []byte{1}); return err }, + func() error { _, err := svc.FinalizeRecording("missing", "ana", 0, RecordingOptions{}); return err }, + func() error { _, err := svc.OpenRecordingAudio("missing"); return err }, + } { + if err := call(); !errors.Is(err, ErrRecordingNotFound) { + t.Fatalf("unknown recording = %v", err) + } + } + if _, err := svc.StartRecording(RecordingStart{MIME: "audio/webm", ConsentVersion: RecordingConsentVersion}, RecordingOptions{}); !errors.Is(err, ErrRecordingBadRequest) { + t.Fatalf("start without user = %v", err) + } + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.OpenRecordingAudio(m.ID); !errors.Is(err, ErrNoAudio) { + t.Fatalf("audio of an active recording = %v", err) + } + if _, err := src.AudioLink(context.Background(), Item{ID: m.ID}); !errors.Is(err, ErrNoAudio) { + t.Fatalf("link of an active recording = %v", err) + } + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(2)); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); err != nil { + t.Fatal(err) + } + if _, err := svc.FinalizeRecording(m.ID, "ana", 0, RecordingOptions{}); !errors.Is(err, ErrRecordingClosed) { + t.Fatalf("finalize twice = %v", err) + } + if _, err := svc.AbortRecording(m.ID, "ana", false, RecordingOptions{}); !errors.Is(err, ErrRecordingClosed) { + t.Fatalf("abort finalized = %v", err) + } + if got := len(src.Store.List()); got != 1 { + t.Fatalf("list = %d", got) + } + // A recording without a fact directory still finalizes. + if err := (&RecordingSource{}).writeFact(RecordingMeta{}); err != nil { + t.Fatalf("writeFact without dir = %v", err) + } +} + +func TestRecordingStoreLoadSkipsJunk(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"not-an-id", "20261116t090000-0000000a", "20261116t090000-0000000b"} { + if err := os.MkdirAll(filepath.Join(dir, name), recordingDirMode); err != nil { + t.Fatal(err) + } + } + _ = os.WriteFile(filepath.Join(dir, "20261116t090000-0000000a", recordingMetaFile), []byte("{not json"), recordingFileMode) + _ = os.WriteFile(filepath.Join(dir, "20261116t090000-0000000b", recordingMetaFile), []byte(`{"id":"other"}`), recordingFileMode) + _ = os.WriteFile(filepath.Join(dir, "stray.txt"), []byte("x"), recordingFileMode) + if got := NewRecordingStore(dir).List(); len(got) != 0 { + t.Fatalf("junk loaded: %+v", got) + } + if got := NewRecordingStore(filepath.Join(dir, "absent")).UsedBytes(); got != 0 { + t.Fatalf("absent dir used = %d", got) + } +} + +func TestMP4BoxWalkEdges(t *testing.T) { + large := make([]byte, mp4LargeBoxHeader+4) + binary.BigEndian.PutUint32(large[:4], 1) + copy(large[4:8], "mdat") + binary.BigEndian.PutUint64(large[8:16], uint64(len(large))) + stream := append(mp4Box("ftyp", []byte("iso5")), large...) + if types, err := MP4TopLevelBoxes(bytes.NewReader(stream)); err != nil || strings.Join(types, ",") != "ftyp,mdat" { + t.Fatalf("64-bit box = %v, %v", types, err) + } + toEnd := append(mp4Box("ftyp", nil), 0, 0, 0, 0, 'm', 'd', 'a', 't', 9, 9) + if types, err := MP4TopLevelBoxes(bytes.NewReader(toEnd)); err != nil || len(types) != 2 { + t.Fatalf("size-0 box = %v, %v", types, err) + } + for name, bad := range map[string][]byte{ + "tiny size": {0, 0, 0, 4, 'f', 't', 'y', 'p'}, + "trailing bytes": append(mp4Box("ftyp", nil), 1, 2, 3), + "bad large size": {0, 0, 0, 1, 'm', 'd', 'a', 't', 0, 0, 0, 0, 0, 0, 0, 2}, + "short large": {0, 0, 0, 1, 'm', 'd', 'a', 't', 0, 0}, + } { + if _, err := MP4TopLevelBoxes(bytes.NewReader(bad)); err == nil { + t.Errorf("%s accepted", name) + } + } + dir := t.TempDir() + webm := filepath.Join(dir, "a.webm") + _ = os.WriteFile(webm, []byte("nope"), recordingFileMode) + if w := checkRecordingContainer(webm, RecordingContainerWebM); w == "" { + t.Fatal("bad webm header passed") + } + mp4 := filepath.Join(dir, "a.mp4") + _ = os.WriteFile(mp4, mp4Box("moov", nil), recordingFileMode) + if w := checkRecordingContainer(mp4, RecordingContainerMP4); w == "" { + t.Fatal("mp4 without ftyp passed") + } + if w := checkRecordingContainer(filepath.Join(dir, "absent"), RecordingContainerMP4); w == "" { + t.Fatal("missing file passed") + } +} + +// reopenOpener counts opens so the retry path is covered. +type reopenOpener struct { + path string + opens int +} + +func (r *reopenOpener) OpenAudio(context.Context, Item) (LocalAudio, error) { + r.opens++ + f, err := os.Open(r.path) + if err != nil { + return LocalAudio{}, err + } + st, _ := f.Stat() + return LocalAudio{File: f, MIME: "audio/webm", Size: st.Size()}, nil +} + +func TestLocalTranscribeAudioReopensForRetries(t *testing.T) { + path := filepath.Join(t.TempDir(), "a.webm") + if err := os.WriteFile(path, webmChunk(12), recordingFileMode); err != nil { + t.Fatal(err) + } + svc, _, _ := newRecordingService(t, enabledRecording()) + op := &reopenOpener{path: path} + rec := &TranscriptRecord{} + job := svc.txEnqueue(Item{Source: "x", ID: "y"}, "fake", "m", transcribe.Speed{}) + defer svc.txDequeue(job) + a, cleanup, err := svc.localTranscribeAudio(context.Background(), Item{ID: "y"}, op, config.InboxTranscription{}, rec, job) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if a.MIME != "audio/webm" || a.Size != 16 || rec.AudioSize != 16 { + t.Fatalf("audio = %+v, rec size %d", a, rec.AudioSize) + } + for i := 0; i < 2; i++ { + rc, err := a.Open(context.Background()) + if err != nil { + t.Fatal(err) + } + n, _ := io.Copy(io.Discard, rc) + _ = rc.Close() + if n != 16 { + t.Fatalf("read %d bytes", n) + } + } + if op.opens != 2 { + t.Fatalf("opens = %d, want the first handle reused then one reopen", op.opens) + } + // An unused first handle is closed by cleanup; an open error is returned. + _, cleanup2, err := svc.localTranscribeAudio(context.Background(), Item{ID: "y"}, op, config.InboxTranscription{}, rec, job) + if err != nil { + t.Fatal(err) + } + cleanup2() + cleanup2() + if _, _, err := svc.localTranscribeAudio(context.Background(), Item{}, &reopenOpener{path: filepath.Join(t.TempDir(), "absent")}, config.InboxTranscription{}, rec, job); err == nil { + t.Fatal("missing file opened") + } + if _, _, err := svc.localTranscribeAudio(context.Background(), Item{}, op, config.InboxTranscription{MaxBytes: 4}, rec, job); !errors.Is(err, transcribe.ErrTooLarge) { + t.Fatalf("over max_bytes = %v", err) + } +} + +func TestRecordingAutoTranscribeSweepActorAndBusy(t *testing.T) { + svc, src, clock := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + var actors []string + m := startRecording(t, svc, "ana", "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, "ana", 0, webmChunk(8)); err != nil { + t.Fatal(err) + } + clock.t = clock.t.Add(3 * time.Hour) + if _, err := svc.RecordingSweep(RecordingOptions{TranscribeAudit: func(a TranscribeAudit) { actors = append(actors, a.Actor) }}); err != nil { + t.Fatal(err) + } + svc.Wait() + if fake.Calls() != 1 || len(actors) != 1 || actors[0] != RecordingSweepActor { + t.Fatalf("idle-finalized recording: calls %d, actors %v", fake.Calls(), actors) + } + // Already transcribed: finalize-time auto-transcribe is a no-op. + got, _ := src.Store.Get(m.ID) + svc.autoTranscribeRecording(src, got, RecordingOptions{}) + svc.Wait() + if fake.Calls() != 1 { + t.Fatal("a transcribed recording was transcribed again") + } + // No actor at all falls back to the auto-transcribe actor. + got.ID = "unknown" + svc.autoTranscribeRecording(src, got, RecordingOptions{}) +} diff --git a/src/pkg/inbox/recordings_transcribe_test.go b/src/pkg/inbox/recordings_transcribe_test.go new file mode 100644 index 0000000000..064805c555 --- /dev/null +++ b/src/pkg/inbox/recordings_transcribe_test.go @@ -0,0 +1,167 @@ +package inbox + +import ( + "context" + "errors" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/knowledge/connector" + "github.com/hivecommons/hive/pkg/transcribe" +) + +// noSelfHTTP fails a test that makes any HTTP call: a recording's audio is +// read from disk, never fetched from the hive's own endpoint. +type noSelfHTTP struct{ t *testing.T } + +func (n noSelfHTTP) RoundTrip(r *http.Request) (*http.Response, error) { + n.t.Errorf("unexpected HTTP request for a hive recording: %s", r.URL) + return nil, errors.New("no HTTP") +} + +func withTranscription(t *testing.T, svc *Service, tx config.InboxTranscription) *transcribe.Fake { + t.Helper() + cfg := svc.Config() + cfg.Transcription = tx + svc.UpdateConfig(cfg) + fake := &transcribe.Fake{Result: transcribe.Result{Segments: []transcribe.Segment{{Text: "decisions were made"}}, Duration: 2 * time.Second}} + svc.NewTranscriber = func(config.InboxTranscription) (transcribe.Transcriber, error) { return fake, nil } + svc.Getenv = func(string) string { return "" } + svc.AudioClient = &http.Client{Transport: noSelfHTTP{t}} + svc.TranscribeBackoff = func(int) time.Duration { return 0 } + return fake +} + +func recordAndFinalize(t *testing.T, svc *Service, user string, opts RecordingOptions) RecordingMeta { + t.Helper() + m := startRecording(t, svc, user, "audio/webm") + if _, err := svc.AppendRecordingChunk(m.ID, user, 0, webmChunk(30)); err != nil { + t.Fatal(err) + } + if _, err := svc.AppendRecordingChunk(m.ID, user, 1, []byte("cluster")); err != nil { + t.Fatal(err) + } + fin, err := svc.FinalizeRecording(m.ID, user, 2000, opts) + if err != nil { + t.Fatal(err) + } + return fin +} + +func TestRecordingAutoTranscribedOnFinalize(t *testing.T) { + svc, src, _ := newRecordingService(t, enabledRecording()) + // Manual mode: a recording is transcribed anyway, once transcription + // is enabled with consent. + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + var audits []TranscribeAudit + m := recordAndFinalize(t, svc, "ana", RecordingOptions{Actor: "ana", TranscribeAudit: func(a TranscribeAudit) { audits = append(audits, a) }}) + svc.Wait() + if fake.Calls() != 1 { + t.Fatalf("transcriber calls = %d, want 1 (auto-transcribe on finalize)", fake.Calls()) + } + if fake.Bytes() != m.Bytes { + t.Fatalf("transcriber read %d bytes, want the %d-byte local file", fake.Bytes(), m.Bytes) + } + rec, ok := svc.transcriptStore().Get(RecordingType, m.ID) + if !ok || rec.Status != TranscriptionDone || !strings.Contains(rec.Text, "decisions were made") || rec.AudioSize != m.Bytes { + t.Fatalf("transcript = %+v, %v", rec, ok) + } + if len(audits) != 1 || audits[0].Actor != "ana" || audits[0].Outcome != TranscriptionDone { + t.Fatalf("transcribe audits = %+v", audits) + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].Pending || items[0].FactState != connector.FactContentMachine { + t.Fatalf("item after transcription = %+v", items) + } + fact, _ := os.ReadFile(items[0].FactPath) + if !strings.Contains(string(fact), "decisions were made") { + t.Fatalf("fact lacks the transcript:\n%s", fact) + } + c, err := svc.Content(context.Background(), items[0]) + if err != nil || !strings.Contains(c.Transcript, "decisions were made") { + t.Fatalf("content = %+v, %v", c, err) + } + _ = src +} + +func TestRecordingNotTranscribedUntilTranscriptionEnabled(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{}) + if svc.RecordingSettings().Transcribes { + t.Fatal("settings say recordings are transcribed while transcription is off") + } + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + svc.Wait() + if fake.Calls() != 0 { + t.Fatal("a recording was transcribed while transcription is disabled") + } + if _, ok := svc.transcriptStore().Get(RecordingType, m.ID); ok { + t.Fatal("a transcript record exists while transcription is disabled") + } + items, _ := svc.Items(context.Background()) + if len(items) != 1 || items[0].TranscribeUnavailable != TranscribeBlockedDisabled { + t.Fatalf("item = %+v, want transcribe_unavailable=disabled", items) + } + // Enabled without consent is not enough (and is rejected by config). + if n, _ := svc.TranscribeSweep(context.Background(), TranscribeOptions{}); n != 0 { + t.Fatalf("sweep without transcription queued %d", n) + } + // Enabled with consent, still manual mode: the sweep picks it up. + withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + n, err := svc.TranscribeSweep(context.Background(), TranscribeOptions{Actor: AutoTranscribeActor}) + svc.Wait() + if err != nil || n != 1 { + t.Fatalf("sweep queued %d, %v; want the recording", n, err) + } + if rec, ok := svc.transcriptStore().Get(RecordingType, m.ID); !ok || rec.Status != TranscriptionDone { + t.Fatalf("transcript after enabling = %+v", rec) + } + if !svc.RecordingSettings().Transcribes { + t.Fatal("settings do not report automatic transcription") + } +} + +func TestRecordingExpiredAudioIsNotTranscribed(t *testing.T) { + svc, _, clock := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{}) + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + clock.t = clock.t.Add(config.DefaultInboxRecordingRetentionDays * 24 * time.Hour) + if res, _ := svc.RecordingSweep(RecordingOptions{}); len(res.Expired) != 1 { + t.Fatalf("not expired: %+v", res) + } + withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true}) + if n, _ := svc.TranscribeSweep(context.Background(), TranscribeOptions{}); n != 0 { + t.Fatalf("sweep queued %d expired recordings", n) + } + svc.Wait() + if fake.Calls() != 0 { + t.Fatal("expired audio was sent to the transcriber") + } + it, ok, _ := svc.Find(context.Background(), RecordingType, m.ID) + if !ok { + t.Fatal("expired recording not listed") + } + if _, err := svc.Transcribe(context.Background(), it, TranscribeOptions{}); err != nil { + t.Fatalf("manual transcribe of expired audio = %v (want a skipped state)", err) + } + rec, _ := svc.transcriptStore().Get(RecordingType, m.ID) + if rec.Status != TranscriptionSkipped || !strings.HasPrefix(rec.Reason, SkipUnsupported) { + t.Fatalf("expired manual transcribe = %+v, want skipped unsupported", rec) + } +} + +func TestRecordingTranscriptionTooLarge(t *testing.T) { + svc, _, _ := newRecordingService(t, enabledRecording()) + fake := withTranscription(t, svc, config.InboxTranscription{Enabled: true, ConsentAcknowledged: true, MaxBytes: 10}) + m := recordAndFinalize(t, svc, "ana", RecordingOptions{}) + svc.Wait() + rec, _ := svc.transcriptStore().Get(RecordingType, m.ID) + if rec.Status != TranscriptionSkipped || !strings.HasPrefix(rec.Reason, SkipTooLarge) { + t.Fatalf("over max_bytes = %+v", rec) + } + _ = fake +} diff --git a/src/pkg/inbox/recordingsource.go b/src/pkg/inbox/recordingsource.go index 09cfca495b..654fb1d81d 100644 --- a/src/pkg/inbox/recordingsource.go +++ b/src/pkg/inbox/recordingsource.go @@ -27,22 +27,6 @@ const ( RecordingSweepActor = "inbox-recording-sweep" ) -// LocalAudio is an audio file on this hive's own disk. -type LocalAudio struct { - File *os.File - Name string - MIME string - Size int64 - ModTime time.Time -} - -// AudioOpener is optionally implemented by sources whose audio lives on this -// hive: playback serves the file directly, and nothing makes an HTTP call to -// the hive itself. -type AudioOpener interface { - OpenAudio(ctx context.Context, it Item) (LocalAudio, error) -} - // RecordingSource lists the recordings made in this hive's dashboard. It is // a builtin source (listed whatever inbox.sources says). Each finalized // recording has a Knowledge fact (personal layer) that starts pending and @@ -81,28 +65,33 @@ func (r *RecordingSource) List(ctx context.Context) ([]Item, error) { if m.State != RecordingFinalized { continue } - it := Item{ - Source: RecordingType, Connector: RecordingConnector, ID: m.ID, Title: m.Title, - RecordedAt: m.CreatedAt, DurationMS: m.DurationMS, Tags: []string{"recording"}, - RecordedBy: m.RecordedBy, AudioExpired: m.AudioExpired(), AudioRetentionDays: days, - Pending: true, FactState: connector.FactContentPending, - } - if m.IncludesTabAudio { - it.Tags = append(it.Tags, "tab-audio") - } - path := r.factPath(m.ID) - if data, err := os.ReadFile(path); err == nil { - fm, body := splitFrontMatter(string(data)) - fact := itemFromFact(fm, body, RecordingLayer, path) - it.FactSlug, it.FactPath, it.Layer, it.FactState = fact.FactSlug, fact.FactPath, fact.Layer, fact.FactState - it.Summary, it.HasTranscript, it.Pending, it.Excerpt = fact.Summary, fact.HasTranscript, fact.Pending, fact.Excerpt - } - out = append(out, it) + out = append(out, r.item(m, days)) } sortItems(out) return out, nil } +// item is the Inbox item of one finalized recording, with its fact. +func (r *RecordingSource) item(m RecordingMeta, retentionDays int) Item { + it := Item{ + Source: RecordingType, Connector: RecordingConnector, ID: m.ID, Title: m.Title, + RecordedAt: m.CreatedAt, DurationMS: m.DurationMS, Tags: []string{"recording"}, + RecordedBy: m.RecordedBy, AudioExpired: m.AudioExpired(), AudioRetentionDays: retentionDays, + Pending: true, FactState: connector.FactContentPending, + } + if m.IncludesTabAudio { + it.Tags = append(it.Tags, "tab-audio") + } + path := r.factPath(m.ID) + if data, err := os.ReadFile(path); err == nil { + fm, body := splitFrontMatter(string(data)) + fact := itemFromFact(fm, body, RecordingLayer, path) + it.FactSlug, it.FactPath, it.Layer, it.FactState = fact.FactSlug, fact.FactPath, fact.Layer, fact.FactState + it.Summary, it.HasTranscript, it.Pending, it.Excerpt = fact.Summary, fact.HasTranscript, fact.Pending, fact.Excerpt + } + return it +} + // Fetch implements Source from the recording's fact. func (r *RecordingSource) Fetch(ctx context.Context, it Item) (Content, error) { provenance := RecordingType + "://recordings/" + it.ID @@ -228,6 +217,9 @@ func (a RecordingAudit) Detail() string { type RecordingOptions struct { Actor string Audit func(RecordingAudit) + // TranscribeAudit receives the inbox_transcribe entry of the automatic + // transcription a finished recording starts. + TranscribeAudit func(TranscribeAudit) } func (o RecordingOptions) audit(action string, fields ...string) { @@ -328,6 +320,33 @@ func (s *Service) recordingFinalized(src *RecordingSource, m RecordingMeta, opts } opts.audit("inbox_recording_finalize", "recording", m.ID, "chunks", strconv.Itoa(m.Chunks), "bytes", strconv.FormatInt(m.Bytes, 10), "duration_ms", strconv.FormatInt(m.DurationMS, 10), "auto", strconv.FormatBool(m.AutoFinalized)) + s.autoTranscribeRecording(src, m, opts) +} + +// autoTranscribeRecording queues the transcription of a finished recording +// with the hive's provider (auto: whisper first). Recording does not stand in +// for the hive-wide transcription consent: nothing runs until +// inbox.transcription is enabled with consent_acknowledged, and then every +// recording counts as opted in whatever inbox.transcription.mode says (the +// transcription sweep picks up recordings finished while it was off). +func (s *Service) autoTranscribeRecording(src *RecordingSource, m RecordingMeta, opts RecordingOptions) { + if !s.Config().Transcription.Active() || m.AudioExpired() { + return + } + it := src.item(m, s.Config().Recording.EffectiveRetentionDays()) + if !it.Pending { + return + } + actor := opts.Actor + if actor == "" { + actor = AutoTranscribeActor + } + st, err := s.Transcribe(context.Background(), it, TranscribeOptions{Actor: actor, Audit: opts.TranscribeAudit}) + if err != nil { + s.Logger.Warn("inbox: auto-transcribe recording", "recording", m.ID, "error", err) + return + } + s.Logger.Info("inbox: recording queued for transcription", "recording", m.ID, "status", st.Status) } // AbortRecording discards an in-progress recording. diff --git a/src/pkg/inbox/transcription.go b/src/pkg/inbox/transcription.go index a0a2781207..943ba8306a 100644 --- a/src/pkg/inbox/transcription.go +++ b/src/pkg/inbox/transcription.go @@ -412,7 +412,7 @@ func (s *Service) runTranscription(it Item, cfg config.InboxTranscription, rec T } case errors.Is(err, transcribe.ErrTooLarge): rec.Status, rec.Reason = TranscriptionSkipped, fmt.Sprintf("%s: the audio is larger than %d MB (inbox.transcription.max_bytes)", SkipTooLarge, cfg.EffectiveMaxBytes()/bytesPerMB) - case errors.Is(err, ErrNoAudio), errors.Is(err, connector.ErrPlaudNoAudio): + case errors.Is(err, ErrNoAudio), errors.Is(err, connector.ErrPlaudNoAudio), errors.Is(err, ErrRecordingExpired): rec.Status, rec.Reason = TranscriptionSkipped, SkipUnsupported+": "+err.Error() default: rec.Status, rec.Reason = TranscriptionFailed, err.Error() @@ -462,6 +462,16 @@ func (s *Service) transcribeItem(ctx context.Context, it Item, cfg config.InboxT if err != nil { return transcribe.Result{}, err } + if op, ok := s.source(it.Source).(AudioOpener); ok { + // Audio kept on this hive (recordings, uploads) is read from disk: + // the spoke never makes an HTTP call to itself. + audio, cleanup, err := s.localTranscribeAudio(ctx, it, op, cfg, rec, job) + if err != nil { + return transcribe.Result{}, err + } + defer cleanup() + return tr.Transcribe(ctx, audio) + } link, err := s.AudioLink(ctx, it) if err != nil { return transcribe.Result{}, err diff --git a/src/pkg/inbox/transcription_sweep.go b/src/pkg/inbox/transcription_sweep.go index 75880e2df7..9fa4f82ca7 100644 --- a/src/pkg/inbox/transcription_sweep.go +++ b/src/pkg/inbox/transcription_sweep.go @@ -32,7 +32,12 @@ const ( func (s *Service) TranscribeSweep(ctx context.Context, opts TranscribeOptions) (queued int, err error) { cfg := s.Config().Transcription store := s.transcriptStore() - if store == nil || !cfg.Active() || cfg.EffectiveMode() != config.InboxTranscriptionModeAuto { + // Dashboard recordings are transcribed whatever the mode: recording one + // opts it in, once transcription is enabled with consent. + if store == nil || !cfg.Active() { + return 0, nil + } + if cfg.EffectiveMode() != config.InboxTranscriptionModeAuto && s.recordingSource() == nil { return 0, nil } items, err := s.Items(ctx) @@ -45,7 +50,8 @@ func (s *Service) TranscribeSweep(ctx context.Context, opts TranscribeOptions) ( if itemsLeft <= 0 { break } - if seen[it.Key()] || !it.Transcribable || !(cfg.AutoSource(it.Source) || cfg.AutoSource(it.Connector)) || !autoDue(it.Transcription, now) { + optedIn := it.Source == RecordingType || cfg.AutoSource(it.Source) || cfg.AutoSource(it.Connector) + if seen[it.Key()] || !it.Transcribable || !optedIn || !autoDue(it.Transcription, now) { continue } seen[it.Key()] = true