Transcribe while recording, and run a hook when a recording starts

CHANGELOG.md

@@ -7,6 +7,20 @@ section here does not ship.
 Releases before 0.3.0 are on the
 [Releases page](https://github.com/TerrifiedBug/yap/releases).
 
+## 0.4.0
+
+The transcript can be read while the meeting is still going.
+
+- `live_transcript`, off by default and a toggle under Recordings in Settings,
+  writes `live.jsonl` in the session folder as the recording runs: one line
+  per chunk, cut when a speaker pauses or every ten seconds. It uses the same
+  loaded model as everything else, so memory is unchanged; the cost is one
+  short inference every few seconds while a call is on. The transcript
+  written at the end is exactly what it was.
+- `on_start` runs a command when a recording begins, the way `on_stop` has
+  always run one when it ends. Both are in Settings now, and both get the
+  session folder as their argument.
+
 ## 0.3.1
 
 Meeting detection now names what has your microphone, and ignoring something

README.md

@@ -93,8 +93,10 @@ what this yap can do. Your own values are never touched.
   "meeting_auto_record": false,
   "meeting_excluded_apps": [],
   "mic_voice_processing": true,
+  "on_start": "my-hook",
   "on_stop": "my-hook",
   "transcription": { "enabled": true },
+  "live_transcript": false,
   "dictation": {
     "model": "parakeet-tdt-ctc-110m",
     "hotkey": "fn",
@@ -108,8 +110,8 @@ what this yap can do. Your own values are never touched.
 
 Save it and yap picks it up. The hotkey, `tap_to_toggle`, the overlay,
 `mute_output`, `newline_after_release`, `meeting_detection`,
-`meeting_auto_record`, `meeting_excluded_apps`, `recordings_dir` and
-`recording_routes` all change on the spot. A new `model` wants a restart, and
+`meeting_auto_record`, `meeting_excluded_apps`, `recordings_dir`,
+`recording_routes` and `live_transcript` all change on the spot. A new `model` wants a restart, and
 yap says so when it sees one.
 
 `hotkey` is a modifier held on its own — `fn`, `rightOption`, `rightCommand`,
@@ -177,8 +179,19 @@ call coming out of your speakers goes back into the mic. Without it, the other
 side gets transcribed twice, the second time as you. If some audio route
 returns silence instead, yap notices inside a second and restarts the mic raw.
 
-`on_stop` runs a command after each recording with the session folder as its
-argument.
+`on_start` runs a command the moment a recording is under way, and `on_stop`
+runs one after each recording. Both get the session folder as their argument.
+
+`live_transcript` writes the transcript while the recording is still going.
+Off by default. On, the session folder gains `live.jsonl`, one JSON object per
+line — `speaker`, `start_ms`, `end_ms`, `text` — appended a few seconds at a
+time as each of you stops talking, or every ten seconds for someone who does
+not. It is the same model dictation uses, so nothing extra is loaded; what it
+costs is one short inference every few seconds for the length of the call,
+and a dictation press during a meeting may wait behind one. The transcript
+written at the end is unchanged and still the one to keep: `live.jsonl` has
+no word timings and breaks where the audio was quietest, not where the
+sentence ended. Pair it with `on_start` and something can read along.
 
 `transcription` is automatic transcription of recordings, on by default.
 Dictation ignores it, since the hotkey always transcribes. When one finishes

Sources/yap/Audio/MicRecorder.swift

@@ -46,6 +46,10 @@ final class MicRecorder: @unchecked Sendable {
     /// Wall-clock time of the first captured buffer — the track's true start,
     /// used to offset-align the two tracks' transcript timestamps.
     private(set) var firstBufferAt: Date?
+    /// A second reader of every mono buffer the file gets, for the live
+    /// transcript. Set before `start()`; called on the tap thread under the
+    /// same lock as the write, so it sees exactly what the file sees.
+    var onSamples: ((AVAudioPCMBuffer) -> Void)?
 
     // Liveness check state (voice-processing path only). Written from the tap
     // callback, read on main when deciding to fall back.
@@ -221,6 +225,7 @@ final class MicRecorder: @unchecked Sendable {
             } catch {
                 warn("mic track write failed: \(error)")
             }
+            self.onSamples?(buffer)
         }
     }
 
@@ -251,6 +256,7 @@ final class MicRecorder: @unchecked Sendable {
             } catch {
                 warn("mic track write failed: \(error)")
             }
+            self.onSamples?(mono)
         }
     }
 

Sources/yap/Audio/SystemAudioRecorder.swift

@@ -38,6 +38,10 @@ final class SystemAudioRecorder {
     /// Wall-clock time of the first captured buffer — the track's true start,
     /// used to offset-align the two tracks' transcript timestamps.
     private(set) var firstBufferAt: Date?
+    /// A second reader of every buffer the file gets, for the live
+    /// transcript. Set before `start()`; called on the tap queue right after
+    /// the write, with the tap's own stereo format.
+    var onSamples: ((AVAudioPCMBuffer) -> Void)?
 
     /// Start capturing system audio, encoding AAC into `url` (use a .caf
     /// extension — CAF needs no finalization pass, so a crash mid-meeting
@@ -153,6 +157,7 @@ final class SystemAudioRecorder {
             } catch {
                 warn("system track write failed: \(error)")
             }
+            self.onSamples?(buffer)
         }
         guard status == noErr, let procID else { throw RecorderError.ioProcCreationFailed(status) }
 

Sources/yap/Config.swift

@@ -6,10 +6,12 @@ import Foundation
 ///       "recordings_dir": "~/Recordings",
 ///       "recording_routes": { "us.zoom.xos": "~/Work" },
 ///       "transcription": { "enabled": true },
+///       "live_transcript": false,
 ///       "mic_voice_processing": true,
 ///       "meeting_detection": false,
 ///       "meeting_auto_record": false,
 ///       "meeting_excluded_apps": [],
+///       "on_start": "my-hook",
 ///       "on_stop": "my-hook",
 ///       "dictation": {
 ///         "model": "parakeet-tdt-ctc-110m",
@@ -21,8 +23,9 @@ import Foundation
 ///       }
 ///     }
 ///
-/// Every key is optional and CLI flags win over the file. `on_stop` is a shell
-/// command spawned with the session directory as its argument — after the
+/// Every key is optional and CLI flags win over the file. `on_start` and
+/// `on_stop` are shell commands spawned with the session directory as their
+/// argument — the first once both tracks are recording, the second after the
 /// transcript is written, or right after recording when transcription is off.
 enum Config {
     static let path = FileManager.default.homeDirectoryForCurrentUser
@@ -56,13 +59,27 @@ enum Config {
         return URL(fileURLWithPath: expanded, isDirectory: true, relativeTo: root).standardizedFileURL
     }
 
+    /// Shell command to spawn the moment a session is recording, or nil.
+    static func onStart() -> String? {
+        guard let cmd = load()?["on_start"] as? String, !cmd.isEmpty else { return nil }
+        return cmd
+    }
+
     /// Shell command to spawn after each session's transcript is written (or
     /// after recording, if transcription is disabled), or nil.
     static func onStop() -> String? {
         guard let cmd = load()?["on_stop"] as? String, !cmd.isEmpty else { return nil }
         return cmd
     }
 
+    /// Whether a session also writes `live.jsonl` as it records, a chunk at
+    /// a time on the same loaded model. Default off: it is one more inference
+    /// every few seconds for the length of a call, and only something reading
+    /// the file as it grows has a use for it. Read at each session start.
+    static func liveTranscript() -> Bool {
+        load()?["live_transcript"] as? Bool ?? false
+    }
+
     /// Whether finished recordings are transcribed automatically. Default on.
     /// Never gates dictation: the hotkey always transcribes.
     static func transcriptionEnabled() -> Bool {
@@ -189,13 +206,15 @@ enum Config {
 
     /// Every value here is the built-in default, so writing this file changes
     /// nothing about how yap behaves — it exists so "Open Config File" has
-    /// something to open and the watcher has something to watch. `on_stop` is
-    /// left out deliberately: there is no sensible default hook.
+    /// something to open and the watcher has something to watch. `on_start`
+    /// and `on_stop` are left out deliberately: there is no sensible default
+    /// hook.
     static let template = """
         {
           "recordings_dir": "~/Recordings",
           "recording_routes": {},
           "transcription": { "enabled": true },
+          "live_transcript": false,
           "mic_voice_processing": true,
           "meeting_detection": false,
           "meeting_auto_record": false,

Sources/yap/Daemon.swift

@@ -615,9 +615,18 @@ final class Daemon: NSObject, NSApplicationDelegate {
             }
             newSession.title = title
             newSession.appBundleID = app?.bundleID
-            try newSession.start()
+            // Read here, not at launch: a toggle in Settings applies to the
+            // next call. The live transcript borrows the daemon's warm model
+            // the same way the post-stop pass does.
+            let live = Config.liveTranscript()
+                ? LiveTranscript(transcriber: transcriber, dir: newSession.dir, startedAt: newSession.startedAt)
+                : nil
+            try newSession.start(live: live)
             session = newSession
             autoStarted = auto
+            if let cmd = Config.onStart() {
+                Hook.run(cmd, dir: newSession.dir) { warn("on_start \($0)") }
+            }
             if !auto {
                 // Started from the menu, so any prompt on screen is offering
                 // something we're already doing — and the pause below would

Sources/yap/Hook.swift

@@ -0,0 +1,24 @@
+import Foundation
+
+/// The two shell hooks from config: `on_start`, given a session folder the
+/// moment its tracks are recording, and `on_stop`, given one whose transcript
+/// is written. Both get the folder as their only argument.
+///
+/// Fire and forget. The process is spawned and never waited on, so a hook that
+/// hangs costs nothing but its own pid; a hook that cannot launch is reported
+/// through `onFailure` and the recorder carries on as if there were none.
+enum Hook {
+    static func run(_ command: String, dir: URL, onFailure: (String) -> Void) {
+        let task = Process()
+        task.executableURL = URL(fileURLWithPath: "/bin/sh")
+        // `$0` rather than interpolation: the folder name can carry a meeting
+        // title with quotes and spaces in it, and the shell parses `command`
+        // exactly once.
+        task.arguments = ["-c", "\(command) \"$0\"", dir.path]
+        do {
+            try task.run()
+        } catch {
+            onFailure("hook failed to launch: \(error)")
+        }
+    }
+}

Sources/yap/RecordingSession.swift

@@ -15,6 +15,9 @@ final class RecordingSession {
 
     private let mic = MicRecorder()
     private let system = SystemAudioRecorder()
+    /// Present only when `live_transcript` was on at start. Fed by both
+    /// recorders, finished after they stop.
+    private var live: LiveTranscript?
     private var stallTimer: Timer?
     /// Tracks already reported, so one stall warns once.
     private var warned: Set<String> = []
@@ -41,8 +44,13 @@ final class RecordingSession {
     }
 
     /// Start both tracks. If the mic fails after the system tap started, the
-    /// tap is torn down so we never run half a session silently.
-    func start(voiceProcessing: Bool = Config.micVoiceProcessing()) throws {
+    /// tap is torn down so we never run half a session silently. With `live`,
+    /// every buffer also feeds the live transcript, which starts ticking once
+    /// both tracks are up.
+    func start(voiceProcessing: Bool = Config.micVoiceProcessing(), live: LiveTranscript? = nil) throws {
+        self.live = live
+        mic.onSamples = live?.sink(for: "me")
+        system.onSamples = live?.sink(for: "them")
         try system.start(writingTo: dir.appendingPathComponent("system.caf"))
         do {
             try mic.start(
@@ -53,6 +61,7 @@ final class RecordingSession {
             system.stop()
             throw error
         }
+        live?.start()
         startStallWatch()
     }
 
@@ -105,6 +114,9 @@ final class RecordingSession {
         stallTimer = nil
         mic.stop()
         system.stop()
+        // After both recorders, so the last cut sees the last buffer.
+        live?.finish()
+        live = nil
 
         let ended = Date()
         let iso = ISO8601DateFormatter()