diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d56e84..0ffc3b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ section here does not ship. Releases before 0.3.0 are on the [Releases page](https://github.com/TerrifiedBug/yap/releases). +## 0.4.0 + +The transcript can be read while the meeting is still going. + +- `live_transcript`, off by default and a toggle under Recordings in Settings, + writes `live.jsonl` in the session folder as the recording runs: one line + per chunk, cut when a speaker pauses or every ten seconds. It uses the same + loaded model as everything else, so memory is unchanged; the cost is one + short inference every few seconds while a call is on. The transcript + written at the end is exactly what it was. +- `on_start` runs a command when a recording begins, the way `on_stop` has + always run one when it ends. Both are in Settings now, and both get the + session folder as their argument. + ## 0.3.1 Meeting detection now names what has your microphone, and ignoring something diff --git a/README.md b/README.md index c622011..94eede5 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,10 @@ what this yap can do. Your own values are never touched. "meeting_auto_record": false, "meeting_excluded_apps": [], "mic_voice_processing": true, + "on_start": "my-hook", "on_stop": "my-hook", "transcription": { "enabled": true }, + "live_transcript": false, "dictation": { "model": "parakeet-tdt-ctc-110m", "hotkey": "fn", @@ -108,8 +110,8 @@ what this yap can do. Your own values are never touched. Save it and yap picks it up. The hotkey, `tap_to_toggle`, the overlay, `mute_output`, `newline_after_release`, `meeting_detection`, -`meeting_auto_record`, `meeting_excluded_apps`, `recordings_dir` and -`recording_routes` all change on the spot. A new `model` wants a restart, and +`meeting_auto_record`, `meeting_excluded_apps`, `recordings_dir`, +`recording_routes` and `live_transcript` all change on the spot. A new `model` wants a restart, and yap says so when it sees one. `hotkey` is a modifier held on its own — `fn`, `rightOption`, `rightCommand`, @@ -177,8 +179,19 @@ call coming out of your speakers goes back into the mic. Without it, the other side gets transcribed twice, the second time as you. If some audio route returns silence instead, yap notices inside a second and restarts the mic raw. -`on_stop` runs a command after each recording with the session folder as its -argument. +`on_start` runs a command the moment a recording is under way, and `on_stop` +runs one after each recording. Both get the session folder as their argument. + +`live_transcript` writes the transcript while the recording is still going. +Off by default. On, the session folder gains `live.jsonl`, one JSON object per +line — `speaker`, `start_ms`, `end_ms`, `text` — appended a few seconds at a +time as each of you stops talking, or every ten seconds for someone who does +not. It is the same model dictation uses, so nothing extra is loaded; what it +costs is one short inference every few seconds for the length of the call, +and a dictation press during a meeting may wait behind one. The transcript +written at the end is unchanged and still the one to keep: `live.jsonl` has +no word timings and breaks where the audio was quietest, not where the +sentence ended. Pair it with `on_start` and something can read along. `transcription` is automatic transcription of recordings, on by default. Dictation ignores it, since the hotkey always transcribes. When one finishes diff --git a/Sources/yap/Audio/MicRecorder.swift b/Sources/yap/Audio/MicRecorder.swift index 3be2581..69ff5f4 100644 --- a/Sources/yap/Audio/MicRecorder.swift +++ b/Sources/yap/Audio/MicRecorder.swift @@ -46,6 +46,10 @@ final class MicRecorder: @unchecked Sendable { /// Wall-clock time of the first captured buffer — the track's true start, /// used to offset-align the two tracks' transcript timestamps. private(set) var firstBufferAt: Date? + /// A second reader of every mono buffer the file gets, for the live + /// transcript. Set before `start()`; called on the tap thread under the + /// same lock as the write, so it sees exactly what the file sees. + var onSamples: ((AVAudioPCMBuffer) -> Void)? // Liveness check state (voice-processing path only). Written from the tap // callback, read on main when deciding to fall back. @@ -221,6 +225,7 @@ final class MicRecorder: @unchecked Sendable { } catch { warn("mic track write failed: \(error)") } + self.onSamples?(buffer) } } @@ -251,6 +256,7 @@ final class MicRecorder: @unchecked Sendable { } catch { warn("mic track write failed: \(error)") } + self.onSamples?(mono) } } diff --git a/Sources/yap/Audio/SystemAudioRecorder.swift b/Sources/yap/Audio/SystemAudioRecorder.swift index fc59999..133abcf 100644 --- a/Sources/yap/Audio/SystemAudioRecorder.swift +++ b/Sources/yap/Audio/SystemAudioRecorder.swift @@ -38,6 +38,10 @@ final class SystemAudioRecorder { /// Wall-clock time of the first captured buffer — the track's true start, /// used to offset-align the two tracks' transcript timestamps. private(set) var firstBufferAt: Date? + /// A second reader of every buffer the file gets, for the live + /// transcript. Set before `start()`; called on the tap queue right after + /// the write, with the tap's own stereo format. + var onSamples: ((AVAudioPCMBuffer) -> Void)? /// Start capturing system audio, encoding AAC into `url` (use a .caf /// extension — CAF needs no finalization pass, so a crash mid-meeting @@ -153,6 +157,7 @@ final class SystemAudioRecorder { } catch { warn("system track write failed: \(error)") } + self.onSamples?(buffer) } guard status == noErr, let procID else { throw RecorderError.ioProcCreationFailed(status) } diff --git a/Sources/yap/Config.swift b/Sources/yap/Config.swift index 30cf3a5..73d1f0f 100644 --- a/Sources/yap/Config.swift +++ b/Sources/yap/Config.swift @@ -6,10 +6,12 @@ import Foundation /// "recordings_dir": "~/Recordings", /// "recording_routes": { "us.zoom.xos": "~/Work" }, /// "transcription": { "enabled": true }, +/// "live_transcript": false, /// "mic_voice_processing": true, /// "meeting_detection": false, /// "meeting_auto_record": false, /// "meeting_excluded_apps": [], +/// "on_start": "my-hook", /// "on_stop": "my-hook", /// "dictation": { /// "model": "parakeet-tdt-ctc-110m", @@ -21,8 +23,9 @@ import Foundation /// } /// } /// -/// Every key is optional and CLI flags win over the file. `on_stop` is a shell -/// command spawned with the session directory as its argument — after the +/// Every key is optional and CLI flags win over the file. `on_start` and +/// `on_stop` are shell commands spawned with the session directory as their +/// argument — the first once both tracks are recording, the second after the /// transcript is written, or right after recording when transcription is off. enum Config { static let path = FileManager.default.homeDirectoryForCurrentUser @@ -56,6 +59,12 @@ enum Config { return URL(fileURLWithPath: expanded, isDirectory: true, relativeTo: root).standardizedFileURL } + /// Shell command to spawn the moment a session is recording, or nil. + static func onStart() -> String? { + guard let cmd = load()?["on_start"] as? String, !cmd.isEmpty else { return nil } + return cmd + } + /// Shell command to spawn after each session's transcript is written (or /// after recording, if transcription is disabled), or nil. static func onStop() -> String? { @@ -63,6 +72,14 @@ enum Config { return cmd } + /// Whether a session also writes `live.jsonl` as it records, a chunk at + /// a time on the same loaded model. Default off: it is one more inference + /// every few seconds for the length of a call, and only something reading + /// the file as it grows has a use for it. Read at each session start. + static func liveTranscript() -> Bool { + load()?["live_transcript"] as? Bool ?? false + } + /// Whether finished recordings are transcribed automatically. Default on. /// Never gates dictation: the hotkey always transcribes. static func transcriptionEnabled() -> Bool { @@ -189,13 +206,15 @@ enum Config { /// Every value here is the built-in default, so writing this file changes /// nothing about how yap behaves — it exists so "Open Config File" has - /// something to open and the watcher has something to watch. `on_stop` is - /// left out deliberately: there is no sensible default hook. + /// something to open and the watcher has something to watch. `on_start` + /// and `on_stop` are left out deliberately: there is no sensible default + /// hook. static let template = """ { "recordings_dir": "~/Recordings", "recording_routes": {}, "transcription": { "enabled": true }, + "live_transcript": false, "mic_voice_processing": true, "meeting_detection": false, "meeting_auto_record": false, diff --git a/Sources/yap/Daemon.swift b/Sources/yap/Daemon.swift index 2d63fcb..d775144 100644 --- a/Sources/yap/Daemon.swift +++ b/Sources/yap/Daemon.swift @@ -615,9 +615,18 @@ final class Daemon: NSObject, NSApplicationDelegate { } newSession.title = title newSession.appBundleID = app?.bundleID - try newSession.start() + // Read here, not at launch: a toggle in Settings applies to the + // next call. The live transcript borrows the daemon's warm model + // the same way the post-stop pass does. + let live = Config.liveTranscript() + ? LiveTranscript(transcriber: transcriber, dir: newSession.dir, startedAt: newSession.startedAt) + : nil + try newSession.start(live: live) session = newSession autoStarted = auto + if let cmd = Config.onStart() { + Hook.run(cmd, dir: newSession.dir) { warn("on_start \($0)") } + } if !auto { // Started from the menu, so any prompt on screen is offering // something we're already doing — and the pause below would diff --git a/Sources/yap/Hook.swift b/Sources/yap/Hook.swift new file mode 100644 index 0000000..74c2b57 --- /dev/null +++ b/Sources/yap/Hook.swift @@ -0,0 +1,24 @@ +import Foundation + +/// The two shell hooks from config: `on_start`, given a session folder the +/// moment its tracks are recording, and `on_stop`, given one whose transcript +/// is written. Both get the folder as their only argument. +/// +/// Fire and forget. The process is spawned and never waited on, so a hook that +/// hangs costs nothing but its own pid; a hook that cannot launch is reported +/// through `onFailure` and the recorder carries on as if there were none. +enum Hook { + static func run(_ command: String, dir: URL, onFailure: (String) -> Void) { + let task = Process() + task.executableURL = URL(fileURLWithPath: "/bin/sh") + // `$0` rather than interpolation: the folder name can carry a meeting + // title with quotes and spaces in it, and the shell parses `command` + // exactly once. + task.arguments = ["-c", "\(command) \"$0\"", dir.path] + do { + try task.run() + } catch { + onFailure("hook failed to launch: \(error)") + } + } +} diff --git a/Sources/yap/RecordingSession.swift b/Sources/yap/RecordingSession.swift index cbad753..4b8d20c 100644 --- a/Sources/yap/RecordingSession.swift +++ b/Sources/yap/RecordingSession.swift @@ -15,6 +15,9 @@ final class RecordingSession { private let mic = MicRecorder() private let system = SystemAudioRecorder() + /// Present only when `live_transcript` was on at start. Fed by both + /// recorders, finished after they stop. + private var live: LiveTranscript? private var stallTimer: Timer? /// Tracks already reported, so one stall warns once. private var warned: Set = [] @@ -41,8 +44,13 @@ final class RecordingSession { } /// Start both tracks. If the mic fails after the system tap started, the - /// tap is torn down so we never run half a session silently. - func start(voiceProcessing: Bool = Config.micVoiceProcessing()) throws { + /// tap is torn down so we never run half a session silently. With `live`, + /// every buffer also feeds the live transcript, which starts ticking once + /// both tracks are up. + func start(voiceProcessing: Bool = Config.micVoiceProcessing(), live: LiveTranscript? = nil) throws { + self.live = live + mic.onSamples = live?.sink(for: "me") + system.onSamples = live?.sink(for: "them") try system.start(writingTo: dir.appendingPathComponent("system.caf")) do { try mic.start( @@ -53,6 +61,7 @@ final class RecordingSession { system.stop() throw error } + live?.start() startStallWatch() } @@ -105,6 +114,9 @@ final class RecordingSession { stallTimer = nil mic.stop() system.stop() + // After both recorders, so the last cut sees the last buffer. + live?.finish() + live = nil let ended = Date() let iso = ISO8601DateFormatter() diff --git a/Sources/yap/Transcription/LiveTranscript.swift b/Sources/yap/Transcription/LiveTranscript.swift new file mode 100644 index 0000000..6331e4c --- /dev/null +++ b/Sources/yap/Transcription/LiveTranscript.swift @@ -0,0 +1,272 @@ +import AVFoundation +import Foundation + +/// The transcript while the recording is still going: `live.jsonl` in the +/// session folder, one JSON object per line, appended as each chunk lands. +/// +/// Same model, same actor as dictation and the post-stop pass, so there is +/// still one loaded model. The recorders hand every buffer here as well as to +/// the track file; each track is resampled to 16 kHz mono and accumulated, +/// and a timer that only exists while a session is live cuts a chunk at the +/// quietest moment and transcribes it. A chunk is a few seconds of one +/// speaker, so each call to the model costs what a dictation press costs. +/// +/// This is a view of the transcript, not the record. The post-stop pass still +/// runs over the whole track with word timings, and `transcript.md` is what +/// it writes; `live.jsonl` is for whatever wants to read along. +/// +/// `@unchecked Sendable` because the audio threads and the timer touch the +/// same accumulators: `lock` is what makes that safe. +final class LiveTranscript: @unchecked Sendable { + /// One line of `live.jsonl`. Property names are the schema. + struct Line: Codable { + let speaker: String + let start_ms: Int + let end_ms: Int + let text: String + } + + private final class Track { + let speaker: String + var converter: AVAudioConverter? + var samples: [Float] = [] + /// Samples already cut, so the next chunk knows where it starts. + var consumed = 0 + /// Wall clock of the first buffer, so chunk times share the session's + /// clock rather than each track's own. + var firstBufferAt: Date? + + init(speaker: String) { self.speaker = speaker } + } + + private let transcriber: any Transcriber + private let startedAt: Date + private let url: URL + private let lock = NSLock() + private var tracks: [String: Track] = [:] + private var timer: Timer? + private var handle: FileHandle? + /// One transcription pass at a time. A tick that lands while the previous + /// chunk is still on the ANE waits for the next tick rather than queuing. + private var inFlight = false + private var finished = false + private let encoder = JSONEncoder() + + private static let target = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: Double(LiveChunker.rate), + channels: 1, + interleaved: false + )! + /// How often the accumulators are looked at. The cost of a tick that + /// finds nothing to cut is a lock and two integer compares. + private static let tickSeconds: TimeInterval = 2 + + init(transcriber: any Transcriber, dir: URL, startedAt: Date) { + self.transcriber = transcriber + self.startedAt = startedAt + self.url = dir.appendingPathComponent("live.jsonl") + } + + /// The buffer sink for one track. Called on that track's audio thread + /// with buffers in the track's own format; resampled here, appended under + /// the lock, nothing else. + func sink(for speaker: String) -> (AVAudioPCMBuffer) -> Void { + let track = Track(speaker: speaker) + lock.lock() + tracks[speaker] = track + lock.unlock() + return { [weak self] buffer in + self?.ingest(buffer, into: track) + } + } + + /// Start the timer. After both tracks are recording, so the first tick + /// has something to look at. + func start() { + FileManager.default.createFile(atPath: url.path, contents: nil) + handle = FileHandle(forWritingAtPath: url.path) + timer = Timer.scheduledTimer(withTimeInterval: Self.tickSeconds, repeats: true) { + [weak self] _ in self?.tick(final: false) + } + } + + /// Stop the timer and transcribe whatever is left. Called after the + /// recorders have stopped, so no buffer can arrive behind the final cut. + func finish() { + timer?.invalidate() + timer = nil + lock.lock() + finished = true + lock.unlock() + tick(final: true) + } + + // MARK: - + + private func ingest(_ buffer: AVAudioPCMBuffer, into track: Track) { + lock.lock() + defer { lock.unlock() } + guard !finished else { return } + if track.firstBufferAt == nil { track.firstBufferAt = Date() } + if track.converter?.inputFormat != buffer.format { + track.converter = AVAudioConverter(from: buffer.format, to: Self.target) + } + guard let converter = track.converter else { return } + + let ratio = Self.target.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64 + guard let out = AVAudioPCMBuffer(pcmFormat: Self.target, frameCapacity: capacity) else { + return + } + var consumed = false + let input: AVAudioConverterInputBlock = { _, status in + if consumed { + status.pointee = .noDataNow + return nil + } + consumed = true + status.pointee = .haveData + return buffer + } + var error: NSError? + let status = converter.convert(to: out, error: &error, withInputFrom: input) + guard status != .error, let data = out.floatChannelData else { return } + track.samples.append( + contentsOf: UnsafeBufferPointer(start: data[0], count: Int(out.frameLength))) + } + + /// Cut what is ready on every track, then transcribe the cuts off the + /// timer's thread. The lock is held only for the cut; the model runs + /// without it so buffers keep landing while a chunk is on the ANE. + private func tick(final: Bool) { + lock.lock() + if inFlight, !final { + lock.unlock() + return + } + var chunks: [(track: Track, samples: [Float], startMs: Int)] = [] + for track in tracks.values { + guard let cut = LiveChunker.cut(track.samples, final: final) else { continue } + let samples = Array(track.samples[.. Int? { + let n = samples.count + if final { return n > 0 ? n : nil } + guard n >= Int(minSeconds * Double(rate)) else { return nil } + + let tail = Int(pauseSeconds * Double(rate)) + if rms(samples[(n - tail)...]) < pauseRMS { return n } + + guard n >= Int(targetSeconds * Double(rate)) else { return nil } + return quietestWindowEnd(samples) + } + + /// The end of the quietest `windowSeconds` span inside the last + /// `searchSeconds`, so the cut lands after the lull rather than in it. + static func quietestWindowEnd(_ samples: [Float]) -> Int { + let n = samples.count + let window = Int(windowSeconds * Double(rate)) + let from = max(0, n - Int(searchSeconds * Double(rate))) + var best = n + var bestEnergy = Float.greatestFiniteMagnitude + var start = from + while start + window <= n { + let energy = rms(samples[start..<(start + window)]) + if energy < bestEnergy { + bestEnergy = energy + best = start + window + } + start += window / 2 + } + return best + } + + static func isSilent(_ samples: [Float]) -> Bool { + var peak: Float = 0 + for s in samples { peak = max(peak, abs(s)) } + return peak < silentPeak + } + + static func rms(_ samples: ArraySlice) -> Float { + guard !samples.isEmpty else { return 0 } + var sum: Double = 0 + for s in samples { sum += Double(s * s) } + return Float((sum / Double(samples.count)).squareRoot()) + } +} diff --git a/Sources/yap/Transcription/TranscriptionCoordinator.swift b/Sources/yap/Transcription/TranscriptionCoordinator.swift index bf4e9f6..bb79bdf 100644 --- a/Sources/yap/Transcription/TranscriptionCoordinator.swift +++ b/Sources/yap/Transcription/TranscriptionCoordinator.swift @@ -231,14 +231,7 @@ actor TranscriptionCoordinator { /// recording when transcription is disabled). private func runHook(for dir: URL) { guard let cmd = Config.onStop() else { return } - let task = Process() - task.launchPath = "/bin/sh" - task.arguments = ["-c", "\(cmd) \"$0\"", dir.path] - do { - try task.run() - } catch { - log(dir, "on_stop hook failed to launch: \(error)") - } + Hook.run(cmd, dir: dir) { log(dir, "on_stop \($0)") } } private func log(_ dir: URL, _ message: String) { diff --git a/Sources/yap/UI/SettingsModel.swift b/Sources/yap/UI/SettingsModel.swift index 59b891d..860453b 100644 --- a/Sources/yap/UI/SettingsModel.swift +++ b/Sources/yap/UI/SettingsModel.swift @@ -49,13 +49,15 @@ final class SettingsModel: ObservableObject { @Published var transcriptionEnabled: Bool { didSet { writeTranscription("enabled", transcriptionEnabled) } } + @Published var liveTranscript: Bool { didSet { write("live_transcript", liveTranscript) } } @Published var micVoiceProcessing: Bool { didSet { write("mic_voice_processing", micVoiceProcessing) } } /// Debounced, unlike every other control: a keystroke is not a decision, /// and writing per character would rewrite the file — and wake the /// watcher — a dozen times while someone types a command. - @Published var onStop: String { didSet { scheduleOnStopWrite() } } + @Published var onStart: String { didSet { scheduleHookWrite("on_start", onStart) } } + @Published var onStop: String { didSet { scheduleHookWrite("on_stop", onStop) } } @Published var meetingDetection: Bool { didSet { write("meeting_detection", meetingDetection) } } @Published var meetingAutoRecord: Bool { @@ -78,7 +80,7 @@ final class SettingsModel: ObservableObject { /// Suppresses the write-through while `init` fills the properties in. private var loading = true - private var onStopWrite: Task? + private var hookWrites: [String: Task] = [:] /// Set while `applyLaunchAtLogin` puts the toggle back after a failure, /// so the reassignment does not re-enter the setter. private var applyingLaunchAtLogin = false @@ -97,7 +99,9 @@ final class SettingsModel: ObservableObject { ?? ModelRegistry.recommended()?.id ?? "" recordingsDir = Self.abbreviated(Config.recordingsDir() ?? Config.defaultRoot) transcriptionEnabled = Config.transcriptionEnabled() + liveTranscript = Config.liveTranscript() micVoiceProcessing = Config.micVoiceProcessing() + onStart = Config.onStart() ?? "" onStop = Config.onStop() ?? "" meetingDetection = Config.meetingDetectionEnabled() meetingAutoRecord = Config.meetingAutoRecord() @@ -276,16 +280,16 @@ final class SettingsModel: ObservableObject { write("recording_routes", Dictionary(uniqueKeysWithValues: routes.map { ($0.id, $0.detail) })) } - private func scheduleOnStopWrite() { + private func scheduleHookWrite(_ key: String, _ text: String) { guard !loading else { return } - onStopWrite?.cancel() - let command = onStop.trimmingCharacters(in: .whitespacesAndNewlines) - onStopWrite = Task { @MainActor [weak self] in + hookWrites[key]?.cancel() + let command = text.trimmingCharacters(in: .whitespacesAndNewlines) + hookWrites[key] = Task { @MainActor [weak self] in try? await Task.sleep(for: .milliseconds(600)) guard !Task.isCancelled else { return } // An empty hook is no hook: drop the key rather than leave an // empty string for `Config.onStop()` to filter out forever. - self?.write("on_stop", command.isEmpty ? nil : command) + self?.write(key, command.isEmpty ? nil : command) } } diff --git a/Sources/yap/UI/SettingsPanes.swift b/Sources/yap/UI/SettingsPanes.swift index 3a3c45f..49228b8 100644 --- a/Sources/yap/UI/SettingsPanes.swift +++ b/Sources/yap/UI/SettingsPanes.swift @@ -119,16 +119,26 @@ struct RecordingPane: View { } Section { Toggle("Transcribe recordings automatically", isOn: $model.transcriptionEnabled) + Toggle("Transcribe while recording", isOn: $model.liveTranscript) Toggle("Voice processing on the mic", isOn: $model.micVoiceProcessing) + } footer: { + Text("While recording writes live.jsonl in the session folder a few seconds at a time, for anything reading along. The transcript written at the end is unchanged.") + .font(.system(size: 11)) + .foregroundStyle(.secondary) } Section { + LabeledContent("Run when a recording starts") { + TextField("shell command", text: $model.onStart) + .font(.system(size: 12, design: .monospaced)) + .textFieldStyle(.roundedBorder) + } LabeledContent("Run after each recording") { TextField("shell command", text: $model.onStop) .font(.system(size: 12, design: .monospaced)) .textFieldStyle(.roundedBorder) } } footer: { - Text("Given the session folder as its argument.") + Text("Each is given the session folder as its argument. The first runs the moment both tracks are recording; the second once the transcript is written.") .font(.system(size: 11)) .foregroundStyle(.secondary) } diff --git a/Sources/yap/Yap.swift b/Sources/yap/Yap.swift index 80aac25..87159db 100644 --- a/Sources/yap/Yap.swift +++ b/Sources/yap/Yap.swift @@ -12,7 +12,7 @@ struct Yap: ParsableCommand { // The single source of truth for what this binary is. Nothing else // holds a version constant, and the release workflow greps this // literal against the tag, so a build can never claim the wrong one. - version: "0.3.1", + version: "0.4.0", // Two subcommands, and that is the product: `run` is the app, `bench` // is the only thing a terminal can do that the menu bar cannot. // Everything else — setup, permissions, the login item, the model, diff --git a/Tests/yapTests/ConfigSerializerTests.swift b/Tests/yapTests/ConfigSerializerTests.swift index 2527635..27c1f56 100644 --- a/Tests/yapTests/ConfigSerializerTests.swift +++ b/Tests/yapTests/ConfigSerializerTests.swift @@ -24,7 +24,8 @@ final class ConfigSerializerTests: XCTestCase { NSDictionary(dictionary: original) ) - let order = ["recordings_dir", "recording_routes", "transcription", "mic_voice_processing", + let order = ["recordings_dir", "recording_routes", "transcription", "live_transcript", + "mic_voice_processing", "meeting_detection", "meeting_auto_record", "meeting_excluded_apps", "dictation"] let offsets = order.map { text.range(of: "\"\($0)\"")?.lowerBound } diff --git a/Tests/yapTests/LiveChunkerTests.swift b/Tests/yapTests/LiveChunkerTests.swift new file mode 100644 index 0000000..ec0ba6e --- /dev/null +++ b/Tests/yapTests/LiveChunkerTests.swift @@ -0,0 +1,53 @@ +import XCTest + +@testable import yap + +/// `LiveChunker.cut` decides where a growing buffer of one speaker breaks. +/// Synthetic audio: a tone stands in for speech, zeros for a pause. +final class LiveChunkerTests: XCTestCase { + private func tone(seconds: Double, amplitude: Float = 0.2) -> [Float] { + let n = Int(seconds * Double(LiveChunker.rate)) + return (0.. [Float] { + [Float](repeating: 0, count: Int(seconds * Double(LiveChunker.rate))) + } + + func testUnderThreeSecondsWaits() { + XCTAssertNil(LiveChunker.cut(tone(seconds: 2.9), final: false)) + } + + func testAPauseTakesEverything() { + let samples = tone(seconds: 4) + silence(seconds: 0.7) + XCTAssertEqual(LiveChunker.cut(samples, final: false), samples.count) + } + + func testSpeechWithNoPauseWaitsForTenSeconds() { + XCTAssertNil(LiveChunker.cut(tone(seconds: 8), final: false)) + } + + func testLongSpeechCutsAfterTheQuietestMoment() { + // Ten seconds of speech with a 300 ms lull two seconds from the end. + let before = tone(seconds: 7.7) + let lull = silence(seconds: 0.3) + let after = tone(seconds: 2.0) + let samples = before + lull + after + let cut = try! XCTUnwrap(LiveChunker.cut(samples, final: false)) + // After the lull, not in it: the window scan steps by half a window, + // so the cut lands within half a window of the lull's end. + let lullEnd = before.count + lull.count + XCTAssertGreaterThanOrEqual(cut, lullEnd - Int(LiveChunker.windowSeconds * Double(LiveChunker.rate)) / 2) + XCTAssertLessThanOrEqual(cut, lullEnd + Int(LiveChunker.windowSeconds * Double(LiveChunker.rate)) / 2) + } + + func testFinalTakesWhateverIsThere() { + XCTAssertEqual(LiveChunker.cut(tone(seconds: 0.5), final: true), Int(0.5 * Double(LiveChunker.rate))) + XCTAssertNil(LiveChunker.cut([], final: true)) + } + + func testSilenceIsSkippedBeforeTheModel() { + XCTAssertTrue(LiveChunker.isSilent(silence(seconds: 1))) + XCTAssertFalse(LiveChunker.isSilent(tone(seconds: 1))) + } +} diff --git a/Tests/yapTests/LiveTranscriptTests.swift b/Tests/yapTests/LiveTranscriptTests.swift new file mode 100644 index 0000000..98b793b --- /dev/null +++ b/Tests/yapTests/LiveTranscriptTests.swift @@ -0,0 +1,102 @@ +import AVFoundation +import XCTest + +@testable import yap + +/// The whole live path on real speech: buffers in through the recorder sink, +/// the timer cutting chunks, the model transcribing them, lines landing in +/// `live.jsonl` while audio is still arriving. +/// +/// Needs the default model on disk and `say` for the speech, so it skips on +/// a runner that has neither. Everything else in the suite runs without a +/// model; this one is the proof that the pieces fit. +final class LiveTranscriptTests: XCTestCase { + func testChunksLandWhileAudioIsStillArriving() throws { + guard let model = ModelRegistry.find("parakeet-tdt-ctc-110m"), ModelStore.isDownloaded(model) + else { throw XCTSkip("default model not downloaded") } + + let dir = FileManager.default.temporaryDirectory + .appendingPathComponent("yap-live-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: dir) } + + let speech = dir.appendingPathComponent("speech.caf") + let say = Process() + say.executableURL = URL(fileURLWithPath: "/usr/bin/say") + say.arguments = [ + "-o", speech.path, "--data-format=LEF32@16000", + "Right, so the main thing from my side this week is the migration. " + + "We have moved the hot tier across and the dashboards are up, " + + "but the alert rules still need porting and that is next sprint. " + + "The one risk is the index moves, which need a change window.", + ] + try say.run() + say.waitUntilExit() + guard say.terminationStatus == 0 else { throw XCTSkip("say unavailable") } + + let file = try AVAudioFile(forReading: speech) + let transcriber = model.makeTranscriber() + let warm = expectation(description: "model warm") + Task { + try await transcriber.warmUp() + warm.fulfill() + } + wait(for: [warm], timeout: 120) + + let live = LiveTranscript(transcriber: transcriber, dir: dir, startedAt: Date()) + let sink = live.sink(for: "me") + live.start() + + // Feed a second at a time, the way a tap would, and let the timer run + // between helpings. Twelve seconds in, a chunk must already be on + // disk — that is the whole point of the feature. + let second = AVAudioFrameCount(file.processingFormat.sampleRate) + func feed(seconds: Int) throws { + for _ in 0.. 0 else { return } + sink(buffer) + } + } + let url = dir.appendingPathComponent("live.jsonl") + func lines() -> [LiveTranscript.Line] { + guard let text = try? String(contentsOf: url, encoding: .utf8) else { return [] } + return text.split(separator: "\n").compactMap { + try? JSONDecoder().decode(LiveTranscript.Line.self, from: Data($0.utf8)) + } + } + func spin(until done: () -> Bool, limit: TimeInterval) { + let deadline = Date().addingTimeInterval(limit) + while !done(), Date() < deadline { + RunLoop.main.run(until: Date().addingTimeInterval(0.1)) + } + } + + try feed(seconds: 12) + spin(until: { !lines().isEmpty }, limit: 15) + let early = lines() + XCTAssertFalse(early.isEmpty, "a chunk lands before the audio has finished arriving") + + try feed(seconds: 30) + live.finish() + spin(until: { lines().count > early.count }, limit: 15) + + let all = lines() + XCTAssertGreaterThan(all.count, early.count, "the final flush adds the tail") + XCTAssertTrue(all.allSatisfy { $0.speaker == "me" }) + XCTAssertEqual(all.map(\.start_ms), all.map(\.start_ms).sorted()) + XCTAssertTrue(all.allSatisfy { $0.end_ms > $0.start_ms }) + let text = all.map(\.text).joined(separator: " ").lowercased() + XCTAssertTrue(text.contains("migration"), "heard the speech: \(text)") + XCTAssertTrue(text.contains("change window"), "heard the tail: \(text)") + + let release = expectation(description: "model released") + Task { + await transcriber.release() + release.fulfill() + } + wait(for: [release], timeout: 30) + } +}