diff --git a/.github/workflows/ocr-review.yml b/.github/workflows/ocr-review.yml new file mode 100644 index 0000000..c9d0817 --- /dev/null +++ b/.github/workflows/ocr-review.yml @@ -0,0 +1,39 @@ +# 阿里开源 OpenCodeReview — PR 自动行级审查 +# https://github.com/alibaba/open-code-review +# +# 前置条件(一次性):仓库 Settings → Secrets and variables → Actions +# 添加 OCR_LLM_AUTH_TOKEN = Z.ai API Key +# 换其它 provider 时改下面三行明文(url / model / llm_use_anthropic)即可。 +name: OpenCodeReview PR Review + +concurrency: + group: ocr-pr-review-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + # pull_request 使 PR 分支上的 workflow 立即生效;same-repo PR 同样可读 secrets。 + # 将来若需 fork PR 触发,改回 pull_request_target 并先把本文件合入 main。 + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + code-review: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Run OpenCodeReview + # pinned to the v1.13.3 tag commit (annotated tag dereferenced) to + # keep the token-bearing step immune to tag hijacking + uses: alibaba/open-code-review@375f64cd6f67a48a135da83cfcf59c52669a09cb # v1.13.3 + with: + llm_url: https://api.z.ai/api/coding/paas/v4/chat/completions + llm_auth_token: ${{ secrets.OCR_LLM_AUTH_TOKEN }} + llm_model: glm-5.3-flash + llm_use_anthropic: 'false' + # GLM 关闭思考输出(上游 action 默认格式) + llm_extra_body: '{"thinking": {"type": "disabled"}}' + language: Chinese diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0a97fe4..c07db3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -191,7 +191,7 @@ jobs: CODE_SIGN_IDENTITY="-" \ CODE_SIGN_STYLE=Automatic \ MACOSX_DEPLOYMENT_TARGET=26.0 \ - ONLY_ACTIVE_ARCH=NO \ + ARCHS=arm64 \ clean build - name: Sign App & Create DMG @@ -220,12 +220,17 @@ jobs: VERSION=${{ steps.vars.outputs.version }} DMG_FILE="${{ env.PROJECT_NAME }}-${VERSION}.dmg" + ARM64_DMG_FILE="${{ env.PROJECT_NAME }}-${VERSION}-arm64.dmg" hdiutil create -volname "${{ env.PROJECT_NAME }}" \ -srcfolder dmg_temp \ -ov -format UDZO \ "$DMG_FILE" + # arm64-only build; keep the un-suffixed name for backward compatibility, + # but release notes state this is an arm64-only artifact (see Create Release body). + cp "$DMG_FILE" "$ARM64_DMG_FILE" + # Get file size DMG_SIZE=$(stat -f%z "$DMG_FILE") echo "dmg_size=$DMG_SIZE" >> $GITHUB_OUTPUT @@ -341,6 +346,8 @@ jobs: ${{ env.PROJECT_NAME }}-*.dmg appcast.xml generate_release_notes: true + body: | + > **注意**:本次构建为 **arm64-only**。`$NAME-$VERSION.dmg` 与 `$NAME-$VERSION-arm64.dmg` 为同一 arm64 产物(无后缀名为向后兼容保留),不含 x86_64 切片。 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 5a01e86..3fd130a 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,4 @@ build_artifacts/ .omc/ .omd/ *.dmg +scripts/e2e/eventsynth diff --git a/TransFrame.xcodeproj/project.pbxproj b/TransFrame.xcodeproj/project.pbxproj index e38209b..d6211e1 100644 --- a/TransFrame.xcodeproj/project.pbxproj +++ b/TransFrame.xcodeproj/project.pbxproj @@ -223,7 +223,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -euo pipefail\n# Sign embedded frameworks with ad-hoc signature to match main app\nFRAMEWORKS_DIR=\"${BUILT_PRODUCTS_DIR}/${PRODUCT_NAME}.app/Contents/Frameworks\"\nif [ -d \"$FRAMEWORKS_DIR\" ]; then\n for FRAMEWORK in \"$FRAMEWORKS_DIR\"/*.framework; do\n if [ -d \"$FRAMEWORK\" ]; then\n echo \"Re-signing framework with ad-hoc: $(basename \"$FRAMEWORK\")\"\n codesign --force --sign - --deep \"$FRAMEWORK\"\n fi\n done\nfi\nmkdir -p \"${DERIVED_FILE_DIR}\"\ntouch \"${DERIVED_FILE_DIR}/TransFrameFrameworkResign.stamp\"\n"; + shellScript = "set -euo pipefail\n# Thin and sign embedded frameworks\nFRAMEWORKS_DIR=\"${BUILT_PRODUCTS_DIR}/${PRODUCT_NAME}.app/Contents/Frameworks\"\nif [ -d \"$FRAMEWORKS_DIR\" ]; then\n TARGET_ARCH=\"${ARCHS:-arm64}\"\n if [ \"$TARGET_ARCH\" = \"arm64\" ] || [ \"$TARGET_ARCH\" = \"x86_64\" ]; then\n if [ -d \"$FRAMEWORKS_DIR/Sparkle.framework\" ]; then\n SPARKLE=\"$FRAMEWORKS_DIR/Sparkle.framework\"\n for b in \"Versions/B/Sparkle\" \"Versions/B/Autoupdate\" \"Versions/B/Updater.app/Contents/MacOS/Updater\" \"Versions/B/XPCServices/Downloader.xpc/Contents/MacOS/Downloader\" \"Versions/B/XPCServices/Installer.xpc/Contents/MacOS/Installer\"; do\n if [ -f \"$SPARKLE/$b\" ]; then\n lipo -thin \"$TARGET_ARCH\" \"$SPARKLE/$b\" -output \"$SPARKLE/$b.thin\" 2>/dev/null && mv \"$SPARKLE/$b.thin\" \"$SPARKLE/$b\" || true\n strip -x \"$SPARKLE/$b\" 2>/dev/null || true\n fi\n done\n fi\n fi\n SIGN_ID=\"${EXPANDED_CODE_SIGN_IDENTITY:-}\"\n if [ -z \"$SIGN_ID\" ] || [ \"$SIGN_ID\" = \"-\" ]; then\n SIGN_ID=$(security find-identity -v -p codesigning | grep -m1 \"Developer ID Application\" | awk -F '\"' '{print $2}' || true)\n fi\n if [ -z \"$SIGN_ID\" ]; then\n SIGN_ID=\"-\"\n fi\n for FRAMEWORK in \"$FRAMEWORKS_DIR\"/*.framework; do\n if [ -d \"$FRAMEWORK\" ]; then\n echo \"Re-signing framework with $SIGN_ID: $(basename \"$FRAMEWORK\")\"\n codesign --force --sign \"$SIGN_ID\" --timestamp=none --deep \"$FRAMEWORK\"\n fi\n done\nfi\nmkdir -p \"${DERIVED_FILE_DIR}\"\ntouch \"${DERIVED_FILE_DIR}/TransFrameFrameworkResign.stamp\"\n"; }; /* End PBXShellScriptBuildPhase section */ @@ -382,11 +382,11 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = "TransFrame/Supporting Files/TransFrame.entitlements"; - CODE_SIGN_IDENTITY = "-"; + CODE_SIGN_IDENTITY = "Developer ID Application"; CODE_SIGN_STYLE = Manual; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 13; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = 27LU47J28S; ENABLE_APP_SANDBOX = NO; ENABLE_HARDENED_RUNTIME = NO; ENABLE_PREVIEWS = YES; @@ -416,15 +416,17 @@ SC000015 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - ARCHS = "$(ARCHS_STANDARD)"; + ARCHS = arm64; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = "TransFrame/Supporting Files/TransFrame.entitlements"; - CODE_SIGN_IDENTITY = "-"; + CODE_SIGN_IDENTITY = "Developer ID Application"; CODE_SIGN_STYLE = Manual; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 13; - DEVELOPMENT_TEAM = ""; + DEAD_CODE_STRIPPING = YES; + DEPLOYMENT_POSTPROCESSING = YES; + DEVELOPMENT_TEAM = 27LU47J28S; ENABLE_APP_SANDBOX = NO; ENABLE_HARDENED_RUNTIME = NO; ENABLE_PREVIEWS = YES; @@ -445,6 +447,8 @@ PRODUCT_MODULE_NAME = TransFrame; PRODUCT_NAME = TransFrame; PROVISIONING_PROFILE_SPECIFIER = ""; + STRIP_INSTALLED_PRODUCT = YES; + STRIP_STYLE = all; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_STRICT_CONCURRENCY = complete; SWIFT_VERSION = 6.0; diff --git a/TransFrame/App/AppDelegate.swift b/TransFrame/App/AppDelegate.swift index 772372a..f1a727d 100644 --- a/TransFrame/App/AppDelegate.swift +++ b/TransFrame/App/AppDelegate.swift @@ -21,7 +21,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { // MARK: - Other Properties private var menuBarController: MenuBarController? - private let settings = AppSettings.shared + private var settings: AppSettings { AppSettings.shared } private lazy var updaterController: SPUStandardUpdaterController = { let controller = SPUStandardUpdaterController( @@ -210,8 +210,87 @@ final class AppDelegate: NSObject, NSApplicationDelegate { HistoryWindowController.shared.showHistory() } + // MARK: - Color Picker & Image Editing Actions + + /// Triggers the native macOS color picker to copy a hex color code + @objc func pickScreenColor() { + NSColorSampler().show { selectedColor in + guard let color = selectedColor else { return } + guard let rgbColor = color.usingColorSpace(.sRGB) else { return } + let r = Int(round(rgbColor.redComponent * 255)) + let g = Int(round(rgbColor.greenComponent * 255)) + let b = Int(round(rgbColor.blueComponent * 255)) + let hex = String(format: "#%02X%02X%02X", r, g, b) + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(hex, forType: .string) + NotificationCenter.default.post(name: .colorDidPick, object: hex) + } + } + + /// Opens an image file from disk and launches the annotation editor + @objc func openImageForEditing() { + let panel = NSOpenPanel() + panel.allowedContentTypes = [.image, .png, .jpeg, .tiff] + panel.allowsMultipleSelection = false + panel.canChooseDirectories = false + panel.canCreateDirectories = false + if panel.runModal() == .OK, let url = panel.url { + openImageURL(url) + } + } + + /// Loads an image file URL and shows the preview editor window + func openImageURL(_ url: URL) { + guard let nsImage = NSImage(contentsOf: url) else { + showImageLoadFailure(url) + return + } + var proposedRect = NSRect(origin: .zero, size: nsImage.size) + guard let cgImage = nsImage.cgImage(forProposedRect: &proposedRect, context: nil, hints: nil) else { + showImageLoadFailure(url) + return + } + + guard let screen = NSScreen.main ?? NSScreen.screens.first else { + showImageLoadFailure(url) + return + } + let displayInfo = DisplayInfo( + id: CGMainDisplayID(), + name: screen.localizedName, + frame: screen.frame, + scaleFactor: screen.backingScaleFactor, + isPrimary: true + ) + let screenshot = Screenshot( + image: cgImage, + sourceDisplay: displayInfo, + format: AppSettings.shared.defaultFormat + ) + PreviewWindowController.shared.showPreview(for: screenshot) + } + + func application(_ application: NSApplication, open urls: [URL]) { + if let first = urls.first { + openImageURL(first) + } + } + // MARK: - Error Handling + /// Shows an alert when an image file cannot be loaded for editing + private func showImageLoadFailure(_ url: URL) { + Logger.general.error("Failed to load image for editing: \(url.path, privacy: .public)") + + let alert = NSAlert() + alert.alertStyle = .warning + alert.messageText = String( + format: NSLocalizedString("error.image.load.failed", comment: ""), + url.lastPathComponent + ) + alert.runModal() + } + /// Shows an error alert for capture failures func showCaptureError(_ error: TransFrameError) { Logger.general.error("Capture error: \(error.localizedDescription)") diff --git a/TransFrame/App/Coordinators/CaptureCoordinator.swift b/TransFrame/App/Coordinators/CaptureCoordinator.swift index bf7726d..ad2877d 100644 --- a/TransFrame/App/Coordinators/CaptureCoordinator.swift +++ b/TransFrame/App/Coordinators/CaptureCoordinator.swift @@ -51,20 +51,20 @@ final class CaptureCoordinator { private func setupSessionCallbacks() { let session = CaptureSessionController.shared - session.onCaptureComplete = { [weak self] document in + session.onCaptureComplete = { document in PreviewWindowController.shared.showPreview(for: document.toLegacyScreenshot()) } - session.onTranslationRequested = { [weak self] document in - let scale = document.sourceDisplay?.scaleFactor ?? 2.0 - TranslationFlowController.shared.startTranslation( - document: document, - scaleFactor: scale + session.onTranslationRequested = { document in + // Route into the preview window's embedded side panel — no external window + PreviewWindowController.shared.showPreview( + for: document.toLegacyScreenshot(), + autoTriggerTranslate: true ) } - session.onOCRRequested = { [weak self] document in - PreviewWindowController.shared.showPreview(for: document.toLegacyScreenshot()) + session.onOCRRequested = { document in + PreviewWindowController.shared.showPreview(for: document.toLegacyScreenshot(), autoTriggerOCR: true) } } diff --git a/TransFrame/Errors/TransFrameError.swift b/TransFrame/Errors/TransFrameError.swift index 84845eb..0cac173 100644 --- a/TransFrame/Errors/TransFrameError.swift +++ b/TransFrame/Errors/TransFrameError.swift @@ -29,6 +29,9 @@ enum TransFrameError: LocalizedError, Sendable { /// Failed to encode the image to the specified format case exportEncodingFailed(format: ExportFormat) + /// Image upload to the configured host failed (network/HTTP/provider error) + case imageUploadFailed(reason: String) + // MARK: - Clipboard Errors /// Failed to write the screenshot to the system clipboard @@ -44,9 +47,6 @@ enum TransFrameError: LocalizedError, Sendable { // MARK: - OCR Errors - /// OCR operation is currently in progress - case ocrOperationInProgress - /// The image provided for OCR is invalid case ocrInvalidImage @@ -74,14 +74,14 @@ enum TransFrameError: LocalizedError, Sendable { return NSLocalizedString("error.disk.full", comment: "") case .exportEncodingFailed: return NSLocalizedString("error.export.encoding.failed", comment: "") + case .imageUploadFailed(let reason): + return String(format: NSLocalizedString("error.image.upload.failed", comment: ""), reason) case .clipboardWriteFailed: return NSLocalizedString("error.clipboard.write.failed", comment: "") case .hotkeyRegistrationFailed: return NSLocalizedString("error.hotkey.registration.failed", comment: "") case .hotkeyConflict: return NSLocalizedString("error.hotkey.conflict", comment: "") - case .ocrOperationInProgress: - return NSLocalizedString("error.ocr.in.progress", comment: "") case .ocrInvalidImage: return NSLocalizedString("error.ocr.invalid.image", comment: "") case .ocrRecognitionFailed: @@ -107,14 +107,14 @@ enum TransFrameError: LocalizedError, Sendable { return NSLocalizedString("error.disk.full.recovery", comment: "") case .exportEncodingFailed: return NSLocalizedString("error.export.encoding.failed.recovery", comment: "") + case .imageUploadFailed: + return NSLocalizedString("error.image.upload.failed.recovery", comment: "") case .clipboardWriteFailed: return NSLocalizedString("error.clipboard.write.failed.recovery", comment: "") case .hotkeyRegistrationFailed: return NSLocalizedString("error.hotkey.registration.failed.recovery", comment: "") case .hotkeyConflict: return NSLocalizedString("error.hotkey.conflict.recovery", comment: "") - case .ocrOperationInProgress: - return NSLocalizedString("error.ocr.in.progress.recovery", comment: "") case .ocrInvalidImage: return NSLocalizedString("error.ocr.invalid.image.recovery", comment: "") case .ocrRecognitionFailed: diff --git a/TransFrame/Extensions/SelectableTextBlock.swift b/TransFrame/Extensions/SelectableTextBlock.swift new file mode 100644 index 0000000..87809b7 --- /dev/null +++ b/TransFrame/Extensions/SelectableTextBlock.swift @@ -0,0 +1,221 @@ +import SwiftUI +import AppKit + +/// Selectable, read-only text block backed by NSTextView. +/// Supports continuous selection across lines (unlike per-row SwiftUI Text) +/// and copies a whole line to the pasteboard on a plain click (PRD 5.5). +struct SelectableTextBlock: NSViewRepresentable { + let attributedText: NSAttributedString + /// Called after a plain click copied one line; used for UI feedback. + var onLineCopied: ((String) -> Void)? = nil + + func makeCoordinator() -> Coordinator { + Coordinator() + } + + func makeNSView(context: Context) -> NSScrollView { + Self.makeScrollView( + textView: Self.makeConfiguredTextView(onLineCopied: onLineCopied) + ) + } + + func updateNSView(_ scroll: NSScrollView, context: Context) { + guard let view = scroll.documentView as? ResultsTextView else { return } + view.onLineCopied = onLineCopied + // Plain string compare avoids rebuilding an attributed copy every + // SwiftUI update pass; content is fully replaced on change. + if context.coordinator.lastAppliedString != attributedText.string { + context.coordinator.lastAppliedString = attributedText.string + view.textStorage?.setAttributedString(attributedText) + // Fresh results invalidate any prior selection instead of + // restoring a stale range over different text. + view.setSelectedRange(NSRange(location: 0, length: 0)) + view.fitToViewport(width: scroll.contentView.bounds.width) + } + } + + final class Coordinator { + var lastAppliedString: String? + } + + /// Read-only selectable text view configured to wrap at the scroll + /// view's width and grow vertically with its content, so long results + /// stay scrollable instead of being clipped. + static func makeConfiguredTextView(onLineCopied: ((String) -> Void)? = nil) -> ResultsTextView { + let textView = ResultsTextView() + textView.drawsBackground = false + textView.isEditable = false + textView.isSelectable = true + textView.isRichText = true + textView.allowsUndo = false + textView.textContainerInset = .zero + textView.textContainer?.lineFragmentPadding = 0 + textView.isVerticallyResizable = true + textView.isHorizontallyResizable = false + textView.textContainer?.widthTracksTextView = true + textView.textContainer?.heightTracksTextView = false + textView.textContainer?.containerSize = NSSize(width: 0, height: CGFloat.greatestFiniteMagnitude) + textView.autoresizingMask = [.width] + textView.onLineCopied = onLineCopied + return textView + } + + /// Native NSScrollView keeps drag/selection gestures in AppKit — + /// SwiftUI's ScrollView would steal them before NSTextView sees them. + static func makeScrollView(textView: ResultsTextView) -> NSScrollView { + let scroll = NSScrollView() + scroll.documentView = textView + scroll.drawsBackground = false + scroll.hasVerticalScroller = true + scroll.hasHorizontalScroller = false + scroll.autohidesScrollers = true + scroll.borderType = .noBorder + return scroll + } +} + +// MARK: - ResultsTextView + +/// Read-only text view: drag selects across lines; a plain click copies the +/// clicked line to the pasteboard. +final class ResultsTextView: NSTextView { + var onLineCopied: ((String) -> Void)? + + private var clickOrigin: NSPoint? + private var pendingLineCopy: DispatchWorkItem? + /// Pasteboard changeCount captured when the deferred copy was scheduled; + /// if anything else writes to the pasteboard during the double-click + /// delay, the deferred copy must not clobber it. + private var pasteboardChangeCountAtClick: Int = 0 + + override func mouseDown(with event: NSEvent) { + // A second/third click cancels the pending single-click line copy: + // the word/paragraph selection must not clobber the pasteboard first. + if event.clickCount >= 2 { + pendingLineCopy?.cancel() + pendingLineCopy = nil + } + clickOrigin = event.locationInWindow + super.mouseDown(with: event) + } + + override func mouseUp(with event: NSEvent) { + super.mouseUp(with: event) + defer { clickOrigin = nil } + + guard let origin = clickOrigin else { return } + let travel = hypot(event.locationInWindow.x - origin.x, event.locationInWindow.y - origin.y) + guard Self.shouldCopyLine( + clickCount: event.clickCount, + modifierFlags: event.modifierFlags, + travel: travel + ) else { return } + + let local = convert(event.locationInWindow, from: nil) + let index = characterIndexForInsertion(at: local) + guard let line = Self.lineString(at: index, in: string) else { return } + + // Defer past the double-click interval: only a true single click + // copies the line. + pasteboardChangeCountAtClick = NSPasteboard.general.changeCount + let work = DispatchWorkItem { [weak self] in + guard let self else { return } + let pasteboard = NSPasteboard.general + // Skip if another app (or a second click here) wrote to the + // pasteboard during the delay — never clobber user data. + guard pasteboard.changeCount == self.pasteboardChangeCountAtClick else { return } + pasteboard.clearContents() + pasteboard.setString(line, forType: .string) + self.onLineCopied?(line) + } + pendingLineCopy?.cancel() + pendingLineCopy = work + DispatchQueue.main.asyncAfter( + deadline: .now() + NSEvent.doubleClickInterval, + execute: work + ) + } + + override func setFrameSize(_ newSize: NSSize) { + let widthChanged = abs(newSize.width - frame.width) > 0.5 + super.setFrameSize(newSize) + // Clip-view / SwiftUI resizes change the wrap width: re-measure the + // content height so long results stay fully visible via scrolling. + if widthChanged { + fitToViewport(width: newSize.width) + } + } + + /// Whether this release should copy its line: a plain single click only — + /// shift+click extends the selection, drags end in one, and a double + /// click's first release is followed by word selection. + static func shouldCopyLine( + clickCount: Int, + modifierFlags: NSEvent.ModifierFlags, + travel: CGFloat + ) -> Bool { + clickCount == 1 && !modifierFlags.contains(.shift) && travel < 3 + } + + /// Wraps the container to `width` and sets the frame height from the laid + /// out content, so the scroll view scrolls instead of clipping. + func fitToViewport(width: CGFloat) { + guard width > 0, let container = textContainer, let layoutManager else { return } + if abs(frame.width - width) > 0.5 { + super.setFrameSize(NSSize(width: width, height: frame.height)) + } + container.containerSize = NSSize(width: width, height: CGFloat.greatestFiniteMagnitude) + layoutManager.ensureLayout(for: container) + let usedHeight = layoutManager.usedRect(for: container).height + let height = ceil(usedHeight) + textContainerInset.height * 2 + 2 + super.setFrameSize(NSSize(width: width, height: max(height, 1))) + } + + /// Returns the full line containing `characterIndex`, or nil when empty. + static func lineString(at characterIndex: Int, in string: String) -> String? { + guard !string.isEmpty else { return nil } + let ns = string as NSString + let clamped = max(0, min(characterIndex, ns.length - 1)) + var lineStart = 0, lineEnd = 0, contentsEnd = 0 + ns.getLineStart(&lineStart, end: &lineEnd, contentsEnd: &contentsEnd, for: NSRange(location: clamped, length: 0)) + let line = ns.substring(with: NSRange(location: lineStart, length: contentsEnd - lineStart)) + return line.isEmpty ? nil : line + } +} + +// MARK: - Attributed builders + +enum ResultsTextBuilder { + /// OCR tab: one continuous block so selection can span lines. + static func ocrText(lines: [String]) -> NSAttributedString { + NSAttributedString( + string: lines.joined(separator: "\n"), + attributes: [ + .font: NSFont.systemFont(ofSize: 13), + .foregroundColor: NSColor.labelColor + ] + ) + } + + /// Translation tab: 原文/译文对照 in one block (styles preserved). + static func translationText(segments: [BilingualSegment]) -> NSAttributedString { + let result = NSMutableAttributedString() + let bodyAttrs: [NSAttributedString.Key: Any] = [ + .font: NSFont.systemFont(ofSize: 13), + .foregroundColor: NSColor.labelColor + ] + let captionAttrs: [NSAttributedString.Key: Any] = [ + .font: NSFont.systemFont(ofSize: 11), + .foregroundColor: NSColor.secondaryLabelColor + ] + for (index, segment) in segments.enumerated() { + if index > 0 { + result.append(NSAttributedString(string: "\n", attributes: bodyAttrs)) + } + result.append(NSAttributedString(string: segment.original.text, attributes: captionAttrs)) + result.append(NSAttributedString(string: "\n", attributes: captionAttrs)) + result.append(NSAttributedString(string: segment.translated, attributes: bodyAttrs)) + } + return result + } +} diff --git a/TransFrame/Extensions/View+Cursor.swift b/TransFrame/Extensions/View+Cursor.swift index 9e62380..76f55ed 100644 --- a/TransFrame/Extensions/View+Cursor.swift +++ b/TransFrame/Extensions/View+Cursor.swift @@ -5,15 +5,9 @@ import AppKit extension View { /// Sets the cursor to use when hovering over this view /// - Parameter cursor: The NSCursor to display - /// - Returns: A view that changes the cursor on hover + /// - Returns: A view that changes the cursor on hover and updates immediately on state changes func cursor(_ cursor: NSCursor) -> some View { - self.onHover { isHovering in - if isHovering { - cursor.push() - } else { - NSCursor.pop() - } - } + self.modifier(CursorViewModifier(cursor: cursor)) } /// Adds scroll wheel zoom handling to the view @@ -25,7 +19,107 @@ extension View { } } -// MARK: - Scroll Wheel Zoom Handler +private struct CursorViewModifier: ViewModifier { + let cursor: NSCursor + @State private var isHovering = false + + func body(content: Content) -> some View { + content + .onHover { hovering in + isHovering = hovering + // Paired push/pop keeps nested .cursor views correct: leaving + // the inner view pops back to the outer cursor instead of + // forcing the arrow. + if hovering { + cursor.push() + } else { + NSCursor.pop() + } + } + .onChange(of: cursor) { _, newCursor in + if isHovering { + NSCursor.pop() + newCursor.push() + } + } + .onDisappear { + if isHovering { + isHovering = false + NSCursor.pop() + } + } + } +} + +// MARK: - NSCursor Symbol & Resize Extension + +extension NSCursor { + nonisolated(unsafe) private static var cursorCache: [String: NSCursor] = [:] + private static let cacheLock = NSLock() + + /// Creates a custom cursor rendered from an SF Symbol with high-contrast white fill and dark outline + static func symbolCursor(systemName: String, hotSpot: NSPoint = NSPoint(x: 10, y: 10)) -> NSCursor? { + let cacheKey = "\(systemName)_\(hotSpot.x)_\(hotSpot.y)" + cacheLock.lock() + if let cached = cursorCache[cacheKey] { + cacheLock.unlock() + return cached + } + cacheLock.unlock() + + let config = NSImage.SymbolConfiguration(pointSize: 14, weight: .semibold) + guard let rawSymbol = NSImage(systemSymbolName: systemName, accessibilityDescription: nil)?.withSymbolConfiguration(config) else { + return nil + } + + let iconSize = NSSize(width: 16, height: 16) + let whiteSymbol = tintedImage(rawSymbol, color: .white, size: iconSize) + let blackSymbol = tintedImage(rawSymbol, color: NSColor.black.withAlphaComponent(0.9), size: iconSize) + + let cursorSize = NSSize(width: 20, height: 20) + let finalImage = NSImage(size: cursorSize, flipped: false) { _ in + let baseRect = NSRect(x: 2, y: 2, width: 16, height: 16) + let offsets: [(CGFloat, CGFloat)] = [ + (-1, 0), (1, 0), (0, -1), (0, 1), + (-0.8, -0.8), (0.8, -0.8), (-0.8, 0.8), (0.8, 0.8) + ] + for (dx, dy) in offsets { + blackSymbol.draw(in: baseRect.offsetBy(dx: dx, dy: dy)) + } + whiteSymbol.draw(in: baseRect) + return true + } + finalImage.isTemplate = false + + let cursor = NSCursor(image: finalImage, hotSpot: hotSpot) + cacheLock.lock() + cursorCache[cacheKey] = cursor + cacheLock.unlock() + return cursor + } + + /// Diagonal resize cursor (NW-SE ↖↘ or NE-SW ↗↙) + static func diagonalResizeCursor(northWestSouthEast: Bool) -> NSCursor { + let symbolName = northWestSouthEast + ? "arrow.up.left.and.arrow.down.right" + : "arrow.up.right.and.arrow.down.left" + if let cursor = symbolCursor(systemName: symbolName, hotSpot: NSPoint(x: 10, y: 10)) { + return cursor + } + return .crosshair + } + + private static func tintedImage(_ image: NSImage, color: NSColor, size: NSSize) -> NSImage { + let tinted = NSImage(size: size, flipped: false) { rect in + image.draw(in: rect) + color.set() + rect.fill(using: .sourceAtop) + return true + } + tinted.isTemplate = false + return tinted + } +} /// NSViewRepresentable that captures scroll wheel events for zooming private struct ScrollWheelZoomView: NSViewRepresentable { diff --git a/TransFrame/Features/Annotations/EllipseTool.swift b/TransFrame/Features/Annotations/EllipseTool.swift index 1b5baf6..931d723 100644 --- a/TransFrame/Features/Annotations/EllipseTool.swift +++ b/TransFrame/Features/Annotations/EllipseTool.swift @@ -1,8 +1,10 @@ import Foundation import CoreGraphics +import AppKit /// Tool for drawing ellipse annotations. /// User drags to define the bounding rectangle of the ellipse. +/// Hold ⌘ while dragging to constrain to a circle. @MainActor struct EllipseTool: AnnotationTool { // MARK: - Properties @@ -16,6 +18,9 @@ struct EllipseTool: AnnotationTool { /// Whether to create filled (solid) ellipses var isFilled: Bool = false + /// Background fill color for filled ellipses (nil = stroke color) + var fillColor: CodableColor? = nil + private var drawingState = DrawingState() // MARK: - AnnotationTool Conformance @@ -26,9 +31,9 @@ struct EllipseTool: AnnotationTool { var currentAnnotation: Annotation? { guard isActive else { return nil } - let rect = calculateRect() + let rect = calculatedRect(constrainSquare: NSEvent.modifierFlags.contains(.command)) guard rect.width > 0 && rect.height > 0 else { return nil } - return .ellipse(EllipseAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled)) + return .ellipse(EllipseAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled, fillColor: fillColor)) } mutating func beginDrawing(at point: CGPoint) { @@ -49,31 +54,36 @@ struct EllipseTool: AnnotationTool { guard isActive else { return nil } continueDrawing(to: point) - let rect = calculateRect() + let rect = calculatedRect(constrainSquare: NSEvent.modifierFlags.contains(.command)) drawingState.reset() guard rect.width >= 5 && rect.height >= 5 else { return nil } - return .ellipse(EllipseAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled)) + return .ellipse(EllipseAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled, fillColor: fillColor)) } mutating func cancelDrawing() { drawingState.reset() } - // MARK: - Private Methods + // MARK: - Internal Methods - private func calculateRect() -> CGRect { + /// Current drag rect. `constrainSquare` mirrors the ⌘ state and is + /// injectable so tests can exercise the 1:1 circle constraint without synthesizing keys. + func calculatedRect(constrainSquare: Bool) -> CGRect { guard drawingState.points.count >= 2 else { return .zero } let start = drawingState.startPoint guard let end = drawingState.points.last else { return .zero } + if constrainSquare { + return GeometryConstraint.squareRect(from: start, to: end) + } + let minX = min(start.x, end.x) let minY = min(start.y, end.y) let maxX = max(start.x, end.x) let maxY = max(start.y, end.y) - return CGRect(x: minX, y: minY, width: maxX - minX, height: maxY - minY) } } diff --git a/TransFrame/Features/Annotations/RectangleTool.swift b/TransFrame/Features/Annotations/RectangleTool.swift index 263d1db..7db5268 100644 --- a/TransFrame/Features/Annotations/RectangleTool.swift +++ b/TransFrame/Features/Annotations/RectangleTool.swift @@ -1,8 +1,10 @@ import Foundation import CoreGraphics +import AppKit /// Tool for drawing rectangle annotations. /// User drags to define opposite corners of the rectangle. +/// Hold ⌘ while dragging to constrain to a square. @MainActor struct RectangleTool: AnnotationTool { // MARK: - Properties @@ -16,6 +18,9 @@ struct RectangleTool: AnnotationTool { /// Whether to create filled (solid) rectangles var isFilled: Bool = false + /// Background fill color for filled rectangles (nil = stroke color) + var fillColor: CodableColor? = nil + private var drawingState = DrawingState() // MARK: - AnnotationTool Conformance @@ -26,9 +31,9 @@ struct RectangleTool: AnnotationTool { var currentAnnotation: Annotation? { guard isActive else { return nil } - let rect = calculateRect() + let rect = calculatedRect(constrainSquare: NSEvent.modifierFlags.contains(.command)) guard rect.width > 0 && rect.height > 0 else { return nil } - return .rectangle(RectangleAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled)) + return .rectangle(RectangleAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled, fillColor: fillColor)) } mutating func beginDrawing(at point: CGPoint) { @@ -50,22 +55,24 @@ struct RectangleTool: AnnotationTool { guard isActive else { return nil } continueDrawing(to: point) - let rect = calculateRect() + let rect = calculatedRect(constrainSquare: NSEvent.modifierFlags.contains(.command)) drawingState.reset() // Only create annotation if it has meaningful size guard rect.width >= 2 && rect.height >= 2 else { return nil } - return .rectangle(RectangleAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled)) + return .rectangle(RectangleAnnotation(rect: rect, style: strokeStyle, isFilled: isFilled, fillColor: fillColor)) } mutating func cancelDrawing() { drawingState.reset() } - // MARK: - Private Methods + // MARK: - Internal Methods - private func calculateRect() -> CGRect { + /// Current drag rect. `constrainSquare` mirrors the ⌘ state and is + /// injectable so tests can exercise the 1:1 constraint without synthesizing keys. + func calculatedRect(constrainSquare: Bool) -> CGRect { guard drawingState.points.count >= 2 else { return .zero } @@ -75,11 +82,14 @@ struct RectangleTool: AnnotationTool { return .zero } + if constrainSquare { + return GeometryConstraint.squareRect(from: start, to: end) + } + let minX = min(start.x, end.x) let minY = min(start.y, end.y) let maxX = max(start.x, end.x) let maxY = max(start.y, end.y) - return CGRect(x: minX, y: minY, width: maxX - minX, height: maxY - minY) } } diff --git a/TransFrame/Features/BilingualResult/BilingualResultView.swift b/TransFrame/Features/BilingualResult/BilingualResultView.swift index 89fa994..7b4c6bc 100644 --- a/TransFrame/Features/BilingualResult/BilingualResultView.swift +++ b/TransFrame/Features/BilingualResult/BilingualResultView.swift @@ -6,34 +6,58 @@ struct BilingualResultView: View { @State private var imageScale: CGFloat = 1.0 var body: some View { - VStack(spacing: 0) { - ScrollView([.horizontal, .vertical], showsIndicators: true) { - Image(decorative: viewModel.image, scale: viewModel.displayScaleFactor) - .resizable() - .aspectRatio(contentMode: .fit) - .scaleEffect(viewModel.scale) - .frame( - width: viewModel.imagePointWidth * viewModel.scale, - height: viewModel.imagePointHeight * viewModel.scale - ) - .onScrollWheelZoom { delta in - if delta > 0 { - viewModel.zoomIn() - } else { - viewModel.zoomOut() + ZStack(alignment: .bottom) { + // Main Canvas Scroll Area + VStack(spacing: 0) { + ScrollView([.horizontal, .vertical], showsIndicators: false) { + Image(decorative: viewModel.image, scale: viewModel.displayScaleFactor) + .resizable() + .aspectRatio(contentMode: .fit) + .scaleEffect(viewModel.scale) + .frame( + width: viewModel.imagePointWidth * viewModel.scale, + height: viewModel.imagePointHeight * viewModel.scale + ) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .padding(16) + .onScrollWheelZoom { delta in + if delta > 0 { + viewModel.zoomIn() + } else { + viewModel.zoomOut() + } } - } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) } - .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color(nsColor: .windowBackgroundColor)) - - Divider() + // Attached Floating Pill Toolbar toolBar - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(.bar) + .cursor(.arrow) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(.regularMaterial) + .clipShape(RoundedRectangle(cornerRadius: 10)) + .shadow(color: .black.opacity(0.25), radius: 8, y: 2) + .padding(.bottom, 12) + } + .overlay(alignment: .topTrailing) { + Button { + BilingualResultWindowController.shared.close() + } label: { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 20)) + .foregroundStyle(.secondary, Color.black.opacity(0.1)) + .padding(10) + } + .buttonStyle(.plain) + .help("Close (Esc)") } + .background( + RoundedRectangle(cornerRadius: 12) + .fill(.ultraThinMaterial) + .shadow(color: .black.opacity(0.35), radius: 16, y: 6) + ) .onKeyPress(.escape) { BilingualResultWindowController.shared.close() return .handled @@ -119,6 +143,21 @@ struct BilingualResultView: View { Label(String(localized: "bilingualResult.save"), systemImage: "square.and.arrow.down") } .buttonStyle(.bordered) + + Button(action: { + // Point size (pixels / displayScaleFactor) so the pin + // renders at on-screen scale instead of raw pixel size. + let pointSize = NSSize( + width: viewModel.imagePointWidth, + height: viewModel.imagePointHeight + ) + let nsImage = NSImage(cgImage: viewModel.image, size: pointSize) + PinnedWindowsManager.shared.pin(image: nsImage) + BilingualResultWindowController.shared.close() + }) { + Label(String(localized: "bilingualResult.pin"), systemImage: "pin") + } + .buttonStyle(.bordered) } } } diff --git a/TransFrame/Features/BilingualResult/BilingualResultWindowController.swift b/TransFrame/Features/BilingualResult/BilingualResultWindowController.swift index 348480a..2b19bfd 100644 --- a/TransFrame/Features/BilingualResult/BilingualResultWindowController.swift +++ b/TransFrame/Features/BilingualResult/BilingualResultWindowController.swift @@ -1,6 +1,11 @@ import AppKit import SwiftUI +final class BilingualResultPanel: NSPanel { + override var canBecomeKey: Bool { true } + override var canBecomeMain: Bool { true } +} + @MainActor final class BilingualResultWindowController: NSObject { static let shared = BilingualResultWindowController() @@ -61,15 +66,23 @@ final class BilingualResultWindowController: NSObject { maxHeight: screenHeight * 0.85 ) - let newWindow = NSWindow( + let newWindow = BilingualResultPanel( contentRect: NSRect(x: 0, y: 0, width: windowSize.width, height: windowSize.height), - styleMask: [.titled, .closable, .miniaturizable, .resizable], + // .titled + .fullSizeContentView keeps the borderless look while + // retaining system edge-resize (a pure .borderless mask ignores + // .resizable for user resizing) and a VoiceOver window title. + styleMask: [.titled, .resizable, .fullSizeContentView], backing: .buffered, defer: false ) + newWindow.titleVisibility = .hidden + newWindow.titlebarAppearsTransparent = true + newWindow.isOpaque = false + newWindow.backgroundColor = .clear + newWindow.hasShadow = true + newWindow.isMovableByWindowBackground = true newWindow.contentView = hostingView - newWindow.title = String(localized: "bilingualResult.window.title") newWindow.center() newWindow.isReleasedWhenClosed = false newWindow.delegate = self @@ -135,15 +148,23 @@ final class BilingualResultWindowController: NSObject { maxHeight: 800 ) - let newWindow = NSWindow( + let newWindow = BilingualResultPanel( contentRect: NSRect(x: 0, y: 0, width: windowSize.width, height: windowSize.height), - styleMask: [.titled, .closable, .miniaturizable, .resizable], + // .titled + .fullSizeContentView keeps the borderless look while + // retaining system edge-resize (a pure .borderless mask ignores + // .resizable for user resizing) and a VoiceOver window title. + styleMask: [.titled, .resizable, .fullSizeContentView], backing: .buffered, defer: false ) + newWindow.titleVisibility = .hidden + newWindow.titlebarAppearsTransparent = true + newWindow.isOpaque = false + newWindow.backgroundColor = .clear + newWindow.hasShadow = true + newWindow.isMovableByWindowBackground = true newWindow.contentView = hostingView - newWindow.title = String(localized: "bilingualResult.window.title") newWindow.center() newWindow.isReleasedWhenClosed = false newWindow.delegate = self diff --git a/TransFrame/Features/Capture/CaptureSessionController.swift b/TransFrame/Features/Capture/CaptureSessionController.swift index 89c7ee3..daf9ba2 100644 --- a/TransFrame/Features/Capture/CaptureSessionController.swift +++ b/TransFrame/Features/Capture/CaptureSessionController.swift @@ -16,6 +16,17 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { case region case screenshotTranslation case fullScreen + + /// Action the overlay runs automatically once a fresh region selection + /// is committed. This is what keeps 截图翻译 distinct from 区域截图. + var automaticActionAfterSelection: SelectionInPlaceToolbar.Action? { + switch self { + case .screenshotTranslation: + return .translate + case .region, .fullScreen: + return nil + } + } } var onCaptureComplete: ((CaptureDocument) -> Void)? @@ -25,6 +36,7 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { private var activeOverlays: [SelectionOverlayWindow] = [] private var currentMode: CaptureMode = .region private var currentSnapshotSet: FrozenSnapshotSet? + private var beautifyConfig: BeautifyConfig? private var localKeyMonitor: Any? private var globalKeyMonitor: Any? @@ -89,6 +101,7 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { snapshotSet: snapshotSet ) overlay.setDelegate(self) + overlay.setAutomaticActionAfterSelection(mode.automaticActionAfterSelection) windows.append(overlay) } @@ -130,6 +143,9 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { return } + let isBeautifyActive = activeOverlays.contains { $0.overlayView?.isBeautifyEnabled == true } + self.beautifyConfig = isBeautifyActive ? BeautifyConfig(isEnabled: true) : nil + // Close overlays immediately closeAllOverlays() @@ -151,10 +167,7 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { ) CaptureDocumentStore.shared.register(doc) - - // If triggered via screenshot translation shortcut, force translate action - let finalAction = (currentMode == .screenshotTranslation && action == .copy) ? .translate : action - dispatchAction(finalAction, for: doc) + dispatchAction(action, for: doc) } catch { logger.error("Failed to crop snapshot: \(error.localizedDescription)") @@ -173,11 +186,16 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { playCaptureSound() let clipboard = ClipboardService() do { - if let rendered = DocumentRenderer.shared.render(document: document) { - try clipboard.copy(rendered, annotations: []) - } else { - try clipboard.copy(document.baseImage, annotations: document.annotations) + let rendered = DocumentRenderer.shared.render(document: document) + var finalImage = rendered ?? document.baseImage + // 渲染失败回退底图时保留标注,交由剪贴板侧合成,避免标注被静默丢弃 + let fallbackAnnotations = rendered == nil ? document.annotations : [] + if let bConfig = beautifyConfig, bConfig.isEnabled { + if let beautified = BeautifyRenderer.shared.render(image: finalImage, config: bConfig) { + finalImage = beautified + } } + try clipboard.copy(finalImage, annotations: fallbackAnnotations) } catch { logger.error("Clipboard copy failed: \(error.localizedDescription)") } @@ -192,13 +210,52 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { case .translate: onTranslationRequested?(document) - case .openEditor: - onCaptureComplete?(document) + case .beautify: + // Handled inside the selection overlay (toggles isBeautifyEnabled); + // the resulting state is read back on confirm and never reaches here. + break + + case .smartMosaic: + // Handled inside the selection overlay (adds mosaic annotations in place) + break case .save: playCaptureSound() - // Forward to preview view model or save directly - onCaptureComplete?(document) + let settings = AppSettings.shared + let exporter = ImageExporter.shared + let directory = settings.saveLocation + let format = settings.defaultFormat + let url = exporter.generateFileURL(in: directory, format: format) + let rendered = DocumentRenderer.shared.render(document: document) + var finalImage = rendered ?? document.baseImage + // 渲染失败回退底图时保留标注,交由导出侧合成,避免存出无标注文件 + let fallbackAnnotations = rendered == nil ? document.annotations : [] + if let bConfig = beautifyConfig, bConfig.isEnabled { + if let beautified = BeautifyRenderer.shared.render(image: finalImage, config: bConfig) { + finalImage = beautified + } + } + do { + try exporter.save(finalImage, annotations: fallbackAnnotations, to: url, format: format, beautifyConfig: nil) + } catch { + logger.error("Save failed: \(error.localizedDescription)") + } + + case .upload: + playCaptureSound() + Task { @MainActor in + var finalImage = DocumentRenderer.shared.render(document: document) ?? document.baseImage + if let bConfig = self.beautifyConfig, bConfig.isEnabled { + if let beautified = BeautifyRenderer.shared.render(image: finalImage, config: bConfig) { + finalImage = beautified + } + } + do { + _ = try await ImageUploadService.shared.upload(image: finalImage) + } catch { + self.logger.error("Upload failed: \(error.localizedDescription)") + } + } case .cancel: break @@ -221,6 +278,10 @@ final class CaptureSessionController: NSObject, SelectionOverlayDelegate { // 1. Local event monitor when app is active localKeyMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in if event.keyCode == 53 { // ESC + if let editingOverlay = self?.activeOverlays.first(where: { $0.overlayView?.isEditingText == true }) { + editingOverlay.overlayView?.cancelTextInput() + return nil + } self?.closeAllOverlays() return nil } diff --git a/TransFrame/Features/Capture/InPlaceResultsPanel.swift b/TransFrame/Features/Capture/InPlaceResultsPanel.swift new file mode 100644 index 0000000..697e5e2 --- /dev/null +++ b/TransFrame/Features/Capture/InPlaceResultsPanel.swift @@ -0,0 +1,249 @@ +import SwiftUI +import AppKit + +/// Shared observable state for the in-place (selection overlay) results drawer. +@MainActor +@Observable +final class InPlaceResultsModel { + enum Tab: Hashable { + case ocr + case translate + } + + var isVisible: Bool = false + var activeTab: Tab = .ocr + var isLoading: Bool = false + var ocrLines: [String] = [] + var segments: [BilingualSegment] = [] + var statusMessage: String? + + /// Notifies the hosting overlay when tab/content changes affect the + /// in-selection bilingual overlay rendering. + var visualInvalidator: (() -> Void)? + + /// Re-run analysis for the active tab + var onRerun: (() -> Void)? + /// Close the drawer + var onClose: (() -> Void)? +} + +/// Right-side results drawer rendered inside the frozen selection overlay. +/// Shows OCR text and bilingual translation without leaving the capture window. +struct InPlaceResultsPanel: View { + @Bindable var model: InPlaceResultsModel + @State private var copiedFlash: String? + @State private var flashGeneration = 0 + + /// Fixed widths so skeleton rows don't jitter on every re-render. + private static let skeletonWidths: [CGFloat] = [220, 170, 200, 140, 210] + + private static let targetLanguages: [TranslationLanguage] = [ + .chineseSimplified, .english, .japanese, .korean, + .french, .german, .spanish, .russian + ] + + var body: some View { + VStack(spacing: 0) { + header + Divider() + Group { + switch model.activeTab { + case .ocr: + ocrContent + case .translate: + translateContent + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + } + .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 8)) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(Color.primary.opacity(0.12), lineWidth: 1) + ) + .onChange(of: model.activeTab) { model.visualInvalidator?() } + .onChange(of: model.segments) { model.visualInvalidator?() } + } + + // MARK: - Header + + private var header: some View { + HStack(spacing: 8) { + if let flash = copiedFlash { + Text(flash) + .font(.caption2) + .foregroundStyle(.green) + .lineLimit(1) + .transition(.opacity) + } + Spacer(minLength: 0) + // 无 tab 切换:内容随工具栏触发动作而定,要另一个结果从工具栏重新点。 + + Button { + model.onRerun?() + } label: { + Image(systemName: "arrow.clockwise") + .font(.system(size: 11)) + .frame(width: 20, height: 20) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.panel.rerun")) + + Button { + copy(currentTabText) + } label: { + Image(systemName: "doc.on.doc") + .font(.system(size: 11)) + .frame(width: 20, height: 20) + } + .buttonStyle(.plain) + .disabled(currentTabText.isEmpty) + .customTooltip(String(localized: "preview.panel.copyAll")) + + Button { + model.onClose?() + } label: { + Image(systemName: "xmark") + .font(.system(size: 11)) + .frame(width: 20, height: 20) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.panel.close")) + } + .padding(.horizontal, 8) + .padding(.vertical, 6) + } + + // MARK: - OCR + + private var ocrContent: some View { + Group { + if model.isLoading && model.activeTab == .ocr { + loadingRows + } else if !model.ocrLines.isEmpty { + SelectableTextBlock( + attributedText: ResultsTextBuilder.ocrText(lines: model.ocrLines), + onLineCopied: flashCopy + ) + .padding(10) + } else { + emptyState + } + } + } + + // MARK: - Translation + + private var translateContent: some View { + Group { + if model.isLoading && model.activeTab == .translate { + loadingRows + } else if !model.segments.isEmpty { + VStack(alignment: .leading, spacing: 8) { + targetLanguagePicker + SelectableTextBlock( + attributedText: ResultsTextBuilder.translationText(segments: model.segments), + onLineCopied: flashCopy + ) + } + .padding(10) + } else { + VStack(spacing: 12) { + emptyState + targetLanguagePicker + Button { + model.onRerun?() + } label: { + Label(String(localized: "preview.panel.translate"), systemImage: "character.book.closed") + } + .buttonStyle(.borderedProminent) + .controlSize(.small) + } + .padding(10) + } + } + } + + private var targetLanguagePicker: some View { + HStack(spacing: 6) { + Text(String(localized: "preview.panel.targetLanguage")) + .font(.caption) + .foregroundStyle(.secondary) + Menu { + ForEach(Self.targetLanguages, id: \.self) { language in + Button { + guard AppSettings.shared.translationTargetLanguage != language else { return } + AppSettings.shared.translationTargetLanguage = language + model.onRerun?() + } label: { + if AppSettings.shared.translationTargetLanguage == language { + Label(language.localizedName, systemImage: "checkmark") + } else { + Text(language.localizedName) + } + } + } + } label: { + Text(AppSettings.shared.translationTargetLanguage?.localizedName ?? "—") + .font(.caption) + } + .controlSize(.small) + .fixedSize() + } + } + + // MARK: - Helpers + + private var currentTabText: String { + switch model.activeTab { + case .ocr: + return model.ocrLines.joined(separator: "\n") + case .translate: + return model.segments.map(\.translated).joined(separator: "\n") + } + } + + private var loadingRows: some View { + VStack(alignment: .leading, spacing: 8) { + ForEach(Self.skeletonWidths, id: \.self) { width in + RoundedRectangle(cornerRadius: 3) + .fill(Color.primary.opacity(0.08)) + .frame(width: width, height: 12) + } + Text(model.activeTab == .ocr + ? String(localized: "preview.panel.recognizing") + : String(localized: "preview.panel.translating")) + .font(.caption) + .foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(10) + } + + private var emptyState: some View { + Text(model.statusMessage ?? String(localized: "preview.panel.noResults")) + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity) + .multilineTextAlignment(.center) + .padding(20) + } + + private func copy(_ text: String) { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(text, forType: .string) + } + + /// Shows the copy feedback; a newer flash invalidates older timers, so + /// rapid line clicks don't clear each other's message early. + private func flashCopy(line: String) { + copiedFlash = String(localized: "preview.panel.copyLine") + ": " + line + flashGeneration += 1 + let generation = flashGeneration + DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { + if flashGeneration == generation { + copiedFlash = nil + } + } + } +} diff --git a/TransFrame/Features/Capture/SelectionInPlaceToolbar.swift b/TransFrame/Features/Capture/SelectionInPlaceToolbar.swift index e44da8f..b04e9ce 100644 --- a/TransFrame/Features/Capture/SelectionInPlaceToolbar.swift +++ b/TransFrame/Features/Capture/SelectionInPlaceToolbar.swift @@ -1,5 +1,6 @@ import AppKit import SwiftUI +import Observation // MARK: - Toolbar Action Protocol @@ -11,42 +12,175 @@ public protocol SelectionToolbarDelegate: AnyObject { func toolbarDidChangeLineWidth(_ width: CGFloat) func toolbarDidRequestUndo() func toolbarDidRequestRedo() + func toolbarDidUpdateLayout() + func toolbarDidChangeMosaicIntensity(_ blockSize: Int) +} + +extension SelectionToolbarDelegate { + public func toolbarDidUpdateLayout() {} + public func toolbarDidChangeMosaicIntensity(_ blockSize: Int) {} +} + +// MARK: - Observable Toolbar State + +@MainActor +@Observable +public final class SelectionToolbarState { + public var activeTool: SelectionInPlaceToolbar.AnnotationTool? + public var activeColor: Color = .red + public var activeLineWidth: CGFloat = 3.0 + public var mosaicBlockSize: Int = 6 + public var showStyleBar: Bool = false + public var isBeautifyEnabled: Bool = false + public var canUndo: Bool = false + public var canRedo: Bool = false + + public init() {} } // MARK: - SelectionInPlaceToolbar /// In-place floating toolbar displayed immediately adjacent to the selected region on frozen overlay. public final class SelectionInPlaceToolbar: NSView { - public enum Action: Sendable { + public enum Action: Equatable, Sendable { case copy case pin case ocr case translate - case openEditor case save + case upload + case beautify + case smartMosaic case cancel } public enum AnnotationTool: String, CaseIterable, Sendable { case rectangle case ellipse + case line case arrow case pen case text + case numberLabel + case highlight case mosaic case privacyRedaction + + public var displayName: String { + switch self { + case .rectangle: return String(localized: "tool.rectangle") + case .ellipse: return String(localized: "tool.ellipse") + case .line: return String(localized: "tool.line") + case .arrow: return String(localized: "tool.arrow") + case .pen: return String(localized: "tool.freehand") + case .text: return String(localized: "tool.text") + case .numberLabel: return String(localized: "tool.numberLabel") + case .highlight: return String(localized: "tool.highlight") + case .mosaic: return String(localized: "tool.mosaic") + case .privacyRedaction: return String(localized: "tool.privacyRedaction") + } + } + + public var shortcutKey: String { + switch self { + case .rectangle: return "R" + case .ellipse: return "O" + case .line: return "L" + case .arrow: return "A" + case .pen: return "D" + case .text: return "T" + case .numberLabel: return "N" + case .highlight: return "H" + case .mosaic: return "M" + case .privacyRedaction: return "X" + } + } + + public var tooltip: String { + "\(displayName) (\(shortcutKey))" + } + + public var cursor: NSCursor { + switch self { + case .text: + return .iBeam + case .pen: + return NSCursor.symbolCursor(systemName: "pencil", hotSpot: NSPoint(x: 3, y: 17)) ?? .crosshair + case .highlight: + return NSCursor.symbolCursor(systemName: "highlighter", hotSpot: NSPoint(x: 3, y: 17)) ?? .crosshair + case .mosaic: + return NSCursor.symbolCursor(systemName: "checkerboard.rectangle", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .privacyRedaction: + return NSCursor.symbolCursor(systemName: "eye.slash.fill", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .numberLabel: + return NSCursor.symbolCursor(systemName: "number.circle.fill", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .rectangle, .ellipse, .line, .arrow: + return .crosshair + } + } } public weak var delegate: SelectionToolbarDelegate? + public let state = SelectionToolbarState() - public private(set) var activeTool: AnnotationTool? - public private(set) var selectedColor: NSColor = .systemRed - public private(set) var selectedLineWidth: CGFloat = 3.0 + public var activeTool: AnnotationTool? { + get { state.activeTool } + set { + state.activeTool = newValue + state.showStyleBar = (newValue != nil) + } + } + + public var selectedColor: NSColor { + get { NSColor(state.activeColor) } + set { state.activeColor = Color(nsColor: newValue) } + } + + public var selectedLineWidth: CGFloat { + get { state.activeLineWidth } + set { state.activeLineWidth = newValue } + } + + /// Ideal content size reported to layout code. + public var preferredSize: CGSize { + let minimum = CGSize(width: 520, height: state.showStyleBar ? 78 : 42) + guard let hosting = hostingView else { return minimum } + let fitted = hosting.fittingSize + guard fitted.width > 1, fitted.height > 1 else { return minimum } + return CGSize( + width: max(fitted.width, minimum.width), + height: max(fitted.height, minimum.height) + ) + } + + /// The unclamped SwiftUI ideal size — exposed so tests can prove the + /// reported `preferredSize` never truncates the button row. + var fittingContentSize: CGSize { + hostingView?.fittingSize ?? .zero + } private var hostingView: NSHostingView? override public var isFlipped: Bool { true } + public override func resetCursorRects() { + super.resetCursorRects() + addCursorRect(bounds, cursor: .arrow) + } + + /// Consume every click within the bar so edge clicks can never fall + /// through to the overlay and cancel/re-pick the selection. Buttons + /// still get their events via `super`; leftover dead zones become + /// inert instead of destructive. + /// + /// - Important: `hitTest(_:)` receives `point` in the **superview's** + /// coordinate system, so containment is checked against `frame` + /// (also superview space) — not `bounds`, which is local. + public override func hitTest(_ point: NSPoint) -> NSView? { + guard frame.contains(point) else { return nil } + return super.hitTest(point) ?? self + } + override public init(frame frameRect: NSRect) { super.init(frame: frameRect) setupViews() @@ -59,6 +193,7 @@ public final class SelectionInPlaceToolbar: NSView { private func setupViews() { wantsLayer = true + clipsToBounds = false layer?.cornerRadius = 8 layer?.masksToBounds = false layer?.shadowColor = NSColor.black.cgColor @@ -67,21 +202,25 @@ public final class SelectionInPlaceToolbar: NSView { layer?.shadowOffset = CGSize(width: 0, height: -2) let content = ToolbarContentView( + state: state, onAction: { [weak self] action in self?.delegate?.toolbarDidSelectAction(action) }, onToolSelected: { [weak self] tool in - self?.activeTool = tool - self?.delegate?.toolbarDidChangeTool(tool) + guard let self else { return } + self.state.activeTool = tool + self.delegate?.toolbarDidChangeTool(tool) + self.delegate?.toolbarDidUpdateLayout() }, onColorSelected: { [weak self] color in - self?.selectedColor = color self?.delegate?.toolbarDidChangeColor(color) }, onLineWidthSelected: { [weak self] width in - self?.selectedLineWidth = width self?.delegate?.toolbarDidChangeLineWidth(width) }, + onMosaicIntensitySelected: { [weak self] blockSize in + self?.delegate?.toolbarDidChangeMosaicIntensity(blockSize) + }, onUndo: { [weak self] in self?.delegate?.toolbarDidRequestUndo() }, @@ -91,6 +230,8 @@ public final class SelectionInPlaceToolbar: NSView { ) let hosting = NSHostingView(rootView: content) + hosting.clipsToBounds = false + hosting.layer?.masksToBounds = false hosting.autoresizingMask = [.width, .height] hosting.frame = bounds addSubview(hosting) @@ -101,7 +242,7 @@ public final class SelectionInPlaceToolbar: NSView { public static func preferredFrame( for selectionRect: CGRect, in bounds: CGRect, - toolbarSize: CGSize = CGSize(width: 480, height: 42) + toolbarSize: CGSize = CGSize(width: 520, height: 42) ) -> CGRect { // Horizontal positioning: // Try aligning right edge of toolbar to right edge of selection @@ -141,17 +282,25 @@ public final class SelectionInPlaceToolbar: NSView { // MARK: - SwiftUI Content View private struct ToolbarContentView: View { + @Bindable var state: SelectionToolbarState let onAction: (SelectionInPlaceToolbar.Action) -> Void let onToolSelected: (SelectionInPlaceToolbar.AnnotationTool?) -> Void let onColorSelected: (NSColor) -> Void let onLineWidthSelected: (CGFloat) -> Void + let onMosaicIntensitySelected: (Int) -> Void let onUndo: () -> Void let onRedo: () -> Void - @State private var activeTool: SelectionInPlaceToolbar.AnnotationTool? - @State private var activeColor: Color = .red - @State private var activeLineWidth: CGFloat = 3.0 - @State private var showStyleBar: Bool = false + private var quickColors: [(color: Color, name: String)] { + [ + (.red, String(localized: "color.red")), + (.yellow, String(localized: "color.yellow")), + (.green, String(localized: "color.green")), + (.blue, String(localized: "color.blue")), + (.black, String(localized: "color.black")), + (.white, String(localized: "color.white")) + ] + } var body: some View { VStack(spacing: 4) { @@ -160,9 +309,12 @@ private struct ToolbarContentView: View { Group { toolButton(.rectangle, icon: "rectangle") toolButton(.ellipse, icon: "circle") + toolButton(.line, icon: "line.diagonal") toolButton(.arrow, icon: "arrow.up.right") toolButton(.pen, icon: "pencil.tip") - toolButton(.text, icon: "textformat") + toolButton(.text, icon: "textbox") + toolButton(.numberLabel, icon: "1.circle.fill") + toolButton(.highlight, icon: "highlighter") toolButton(.mosaic, icon: "checkerboard.rectangle") toolButton(.privacyRedaction, icon: "eye.slash.fill") } @@ -171,21 +323,42 @@ private struct ToolbarContentView: View { // Actions Group { - actionButton("P", icon: "pin.fill", help: "Pin (P)") { + Button { + onAction(.beautify) + } label: { + Image(systemName: "sparkles") + .font(.system(size: 12)) + .foregroundColor(state.isBeautifyEnabled ? .accentColor : .primary) + .padding(5) + .background(state.isBeautifyEnabled ? Color.accentColor.opacity(0.2) : Color.clear) + .cornerRadius(4) + .padding(4) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.tooltip.beautify")) + + actionButton("SM", icon: "camera.metering.unknown", help: String(localized: "preview.tooltip.smartMosaic")) { + onAction(.smartMosaic) + } + + actionButton("P", icon: "pin.fill", help: String(localized: "preview.tooltip.pin")) { onAction(.pin) } - actionButton("OCR", icon: "text.viewfinder", help: "OCR Recognition") { + actionButton("OCR", icon: "text.viewfinder", help: String(localized: "preview.tooltip.ocr")) { onAction(.ocr) } - actionButton("T", icon: "character.book.closed.fill", help: "Translate (T)") { + actionButton("Translate", icon: "character.book.closed.fill", help: "Translate (⌘T)") { onAction(.translate) } - actionButton("Edit", icon: "slider.horizontal.3", help: "Open in Full Editor") { - onAction(.openEditor) - } - actionButton("Save", icon: "square.and.arrow.down", help: "Save (⌘S)") { + actionButton("Save", icon: "square.and.arrow.down", help: String(localized: "preview.tooltip.save")) { onAction(.save) } + if ImageUploadService.shared.isConfigured { + actionButton("Upload", icon: "icloud.and.arrow.up", help: "Upload to Cloud (\(ImageUploadService.shared.settings.activeHost.displayName))") { + onAction(.upload) + } + } } Divider().frame(height: 18) @@ -197,9 +370,11 @@ private struct ToolbarContentView: View { Image(systemName: "xmark") .foregroundColor(.secondary) .padding(5) + .padding(4) + .contentShape(Rectangle()) } .buttonStyle(.plain) - .help("Cancel (Esc)") + .customTooltip("Cancel (Esc)") Button { onAction(.copy) @@ -212,33 +387,90 @@ private struct ToolbarContentView: View { .foregroundColor(.white) .padding(.horizontal, 8) .padding(.vertical, 4) - .background(Color.accentColor) - .cornerRadius(5) + .background(Color.accentColor, in: RoundedRectangle(cornerRadius: 5)) + .padding(3) + .contentShape(Rectangle()) } .buttonStyle(.plain) - .help("Copy to Clipboard (Enter)") + .customTooltip(String(localized: "preview.tooltip.confirm")) } .padding(.horizontal, 8) .padding(.vertical, 6) - .background(.regularMaterial) - .cornerRadius(8) + .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 8)) // Secondary Style Bar for annotations - if showStyleBar { + if state.showStyleBar { HStack(spacing: 8) { // Quick Colors - ForEach([Color.red, Color.yellow, Color.green, Color.blue, Color.black, Color.white], id: \.self) { color in + ForEach(quickColors, id: \.name) { item in Circle() - .fill(color) + .fill(item.color) .frame(width: 14, height: 14) .overlay( Circle() - .stroke(Color.primary.opacity(0.4), lineWidth: activeColor == color ? 2 : 0.5) + .stroke(Color.primary.opacity(0.4), lineWidth: state.activeColor == item.color ? 2 : 0.5) ) + .contentShape(Circle()) .onTapGesture { - activeColor = color - onColorSelected(NSColor(color)) + state.activeColor = item.color + onColorSelected(NSColor(item.color)) } + .customTooltip(item.name, placement: .bottom) + } + + // Screen eyedropper + Button { + NSColorSampler().show { picked in + guard let picked else { return } + state.activeColor = Color(nsColor: picked) + onColorSelected(picked) + } + } label: { + Image(systemName: "eyedropper") + .font(.system(size: 11)) + .padding(3) + } + .buttonStyle(.plain) + .contentShape(Rectangle().inset(by: -2)) + .customTooltip(String(localized: "preview.color.eyedropper"), placement: .bottom) + + // System color picker + ColorPicker( + "", + selection: Binding( + get: { state.activeColor }, + set: { newColor in + state.activeColor = newColor + onColorSelected(NSColor(newColor)) + } + ), + supportsOpacity: false + ) + .labelsHidden() + .controlSize(.small) + .scaleEffect(x: 0.6, y: 0.6) + .frame(width: 20, height: 18) + .contentShape(Rectangle().inset(by: -2)) + .customTooltip(String(localized: "preview.color.custom"), placement: .bottom) + + if state.activeTool == .mosaic { + Divider().frame(height: 12) + + // Mosaic intensity presets (block size) + ForEach([(String(localized: "mosaic.intensity.light"), 6), (String(localized: "mosaic.intensity.medium"), 16), (String(localized: "mosaic.intensity.strong"), 32)], id: \.1) { item in + Text(item.0) + .font(.system(size: 10)) + .padding(.horizontal, 6) + .padding(.vertical, 2) + .background(state.mosaicBlockSize == item.1 ? Color.accentColor.opacity(0.3) : Color.clear) + .cornerRadius(4) + .contentShape(Rectangle()) + .onTapGesture { + state.mosaicBlockSize = item.1 + onMosaicIntensitySelected(item.1) + } + .customTooltip(item.0, placement: .bottom) + } } Divider().frame(height: 12) @@ -248,13 +480,17 @@ private struct ToolbarContentView: View { Circle() .fill(Color.primary) .frame(width: CGFloat(width * 2), height: CGFloat(width * 2)) - .padding(2) - .background(activeLineWidth == width ? Color.accentColor.opacity(0.3) : Color.clear) - .cornerRadius(4) + .padding(3) + .background( + state.activeLineWidth == width ? Color.accentColor.opacity(0.3) : Color.clear, + in: RoundedRectangle(cornerRadius: 4) + ) + .contentShape(Rectangle()) .onTapGesture { - activeLineWidth = width + state.activeLineWidth = width onLineWidthSelected(width) } + .customTooltip("\(String(localized: "settings.stroke.width")): \(Int(width))pt", placement: .bottom) } Spacer() @@ -264,54 +500,76 @@ private struct ToolbarContentView: View { } label: { Image(systemName: "arrow.uturn.backward") .font(.caption) + .padding(3) } .buttonStyle(.plain) + .contentShape(Rectangle().inset(by: -2)) + .disabled(!state.canUndo) + .customTooltip(String(localized: "preview.tooltip.undo"), placement: .bottom) Button { onRedo() } label: { Image(systemName: "arrow.uturn.forward") .font(.caption) + .padding(3) } .buttonStyle(.plain) + .contentShape(Rectangle().inset(by: -2)) + .disabled(!state.canRedo) + .customTooltip(String(localized: "preview.tooltip.redo"), placement: .bottom) } .padding(.horizontal, 10) .padding(.vertical, 4) - .background(.ultraThinMaterial) - .cornerRadius(6) + .background(.ultraThinMaterial, in: RoundedRectangle(cornerRadius: 6)) } } + .contentShape(Rectangle()) + .cursor(.arrow) } private func toolButton(_ tool: SelectionInPlaceToolbar.AnnotationTool, icon: String) -> some View { Button { - if activeTool == tool { - activeTool = nil - showStyleBar = false + if state.activeTool == tool { + state.activeTool = nil + state.showStyleBar = false onToolSelected(nil) } else { - activeTool = tool - showStyleBar = true + state.activeTool = tool + state.showStyleBar = true onToolSelected(tool) } } label: { + // Padding lives inside the label so the whole (larger) area + // triggers the button — edge clicks included. Image(systemName: icon) .font(.system(size: 12)) - .foregroundColor(activeTool == tool ? .accentColor : .primary) + .foregroundColor(state.activeTool == tool ? .accentColor : .primary) .padding(5) - .background(activeTool == tool ? Color.accentColor.opacity(0.2) : Color.clear) - .cornerRadius(4) + .background( + state.activeTool == tool ? Color.accentColor.opacity(0.2) : Color.clear, + in: RoundedRectangle(cornerRadius: 4) + ) + .padding(4) + .contentShape(Rectangle()) } .buttonStyle(.plain) + .customTooltip(tool.tooltip) + .accessibilityLabel(tool.tooltip) } private func actionButton(_ title: String, icon: String, help: String, action: @escaping () -> Void) -> some View { Button(action: action) { + // Padding lives inside the label so the whole (larger) area + // triggers the button — edge clicks included. Image(systemName: icon) .font(.system(size: 12)) .padding(5) + .padding(4) + .contentShape(Rectangle()) } .buttonStyle(.plain) - .help(help) + .customTooltip(help) + .accessibilityLabel(help) } } diff --git a/TransFrame/Features/Capture/SelectionOverlayWindow.swift b/TransFrame/Features/Capture/SelectionOverlayWindow.swift index f0ebf77..a458cdf 100644 --- a/TransFrame/Features/Capture/SelectionOverlayWindow.swift +++ b/TransFrame/Features/Capture/SelectionOverlayWindow.swift @@ -1,5 +1,6 @@ import AppKit import CoreGraphics +import CoreImage import CoreText import SwiftUI import os @@ -82,7 +83,8 @@ final class SelectionOverlayWindow: NSPanel { @MainActor private func configureWindow() { - level = .screenSaver + level = NSWindow.Level(NSWindow.Level.statusBar.rawValue + 1) + sharingType = .readOnly isOpaque = true backgroundColor = .black ignoresMouseEvents = false @@ -122,6 +124,13 @@ final class SelectionOverlayWindow: NSPanel { overlayView?.delegate = delegate } + /// Action to run automatically when the user commits a fresh region + /// selection (e.g. .translate for the screenshot-translation hotkey). + @MainActor + func setAutomaticActionAfterSelection(_ action: SelectionInPlaceToolbar.Action?) { + overlayView?.automaticActionAfterSelection = action + } + @MainActor func showOverlay(isKey: Bool = true) { if isKey { @@ -145,9 +154,9 @@ enum ResizeHandle: CaseIterable { func cursor() -> NSCursor { switch self { case .topLeft, .bottomRight: - return NSCursor(image: NSImage(size: NSSize(width: 16, height: 16)), hotSpot: NSPoint(x: 8, y: 8)) + return NSCursor.diagonalResizeCursor(northWestSouthEast: true) case .topRight, .bottomLeft: - return NSCursor.crosshair + return NSCursor.diagonalResizeCursor(northWestSouthEast: false) case .topCenter, .bottomCenter: return NSCursor.resizeUpDown case .middleLeft, .middleRight: @@ -158,13 +167,17 @@ enum ResizeHandle: CaseIterable { // MARK: - SelectionOverlayView -final class SelectionOverlayView: NSView, SelectionToolbarDelegate { +final class SelectionOverlayView: NSView, SelectionToolbarDelegate, NSTextFieldDelegate { + private static let sharedCIContext = CIContext() + enum InteractionState { case hovering case creatingSelection(start: CGPoint, current: CGPoint) case selected(rect: CGRect) case resizing(handle: ResizeHandle, anchorRect: CGRect, startPoint: CGPoint, currentRect: CGRect) case moving(anchorRect: CGRect, startPoint: CGPoint, currentRect: CGRect) + case movingAnnotation(index: Int, selectionRect: CGRect, startPoint: CGPoint, originalAnno: Annotation, originalAnnotations: [Annotation]) + case resizingAnnotation(index: Int, selectionRect: CGRect, handle: AnnotationHandle, originalAnnotations: [Annotation]) case drawingAnnotation(tool: SelectionInPlaceToolbar.AnnotationTool, selectionRect: CGRect, start: CGPoint, current: CGPoint) } @@ -184,14 +197,41 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { // Annotations in progress private var committedAnnotations: [Annotation] = [] + private var selectedAnnotationIndex: Int? private var activeAnnotationTool: SelectionInPlaceToolbar.AnnotationTool? private var annotationColor: NSColor = .systemRed private var annotationLineWidth: CGFloat = 3.0 + private var mosaicBlockSize: Int = 6 private var currentDraftAnnotation: Annotation? private var freehandPoints: [CGPoint] = [] + // In-place text editing & beautify state + private var activeTextField: NSTextField? + private var textInputPosition: CGPoint? + private var editingAnnotationIndex: Int? + private(set) var isBeautifyEnabled: Bool = false + + var isEditingText: Bool { activeTextField != nil } + private var toolbar: SelectionInPlaceToolbar? private let editHistory = EditHistory() + private var statusLabel: NSTextField? + private var statusHideWorkItem: DispatchWorkItem? + + // In-place results drawer (OCR / translation) + let resultsModel = InPlaceResultsModel() + private var resultsPanelHost: NSHostingView? + /// In-flight drawer analysis; cancelled on rerun and when the drawer closes + /// so a dismissed overlay keeps no running network work writing into it. + private var inPlaceAnalysisTask: Task? + + /// Selection snapshot captured when a click starts outside the current + /// selection; tiny clicks (no drag) must restore it instead of re-picking. + private var selectionBeforeOutsideClick: CGRect? + + /// Action auto-run once after a fresh region selection is committed. + /// Distinguishes screenshot-translation mode from plain region capture. + var automaticActionAfterSelection: SelectionInPlaceToolbar.Action? private let handleSize: CGFloat = 8.0 @@ -251,6 +291,10 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { return currentRect case .moving(_, _, let currentRect): return currentRect + case .movingAnnotation(_, let selRect, _, _, _): + return selRect + case .resizingAnnotation(_, let selRect, _, _): + return selRect case .drawingAnnotation(_, let selRect, _, _): return selRect case .hovering: @@ -279,8 +323,14 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { } context.restoreGState() - // 3. Draw selection border, resize handles, and dimension tag + // 3. Draw selection border, beautify preview, resize handles, and dimension tag if let sel = currentSelection, !sel.isEmpty { + if isBeautifyEnabled { + drawBeautifyPreview(for: sel, context: context) + } + if case .selected = state { + drawBilingualOverlay(for: sel) + } drawSelectionBox(sel, context: context) switch state { case .selected: @@ -294,13 +344,45 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { } // 4. Draw committed and draft annotations - for annotation in committedAnnotations { + for (idx, annotation) in committedAnnotations.enumerated() { + if idx == editingAnnotationIndex { + continue + } drawAnnotation(annotation, context: context) } if let draft = currentDraftAnnotation { drawAnnotation(draft, context: context) } + // 4.1 Draw selection outline and interactive handles around selected annotation + if let selIdx = selectedAnnotationIndex, selIdx < committedAnnotations.count { + let selectedAnno = committedAnnotations[selIdx] + let annoBounds = selectedAnno.bounds.insetBy(dx: -4, dy: -4) + context.saveGState() + context.setStrokeColor(NSColor.controlAccentColor.cgColor) + context.setLineWidth(1.5) + context.setLineDash(phase: 0, lengths: [4, 4]) + context.stroke(annoBounds) + context.setLineDash(phase: 0, lengths: []) + + for item in selectedAnno.controlHandles(padding: 4) { + let isSpecial = item.handle.isSpecialPoint + let hSize: CGFloat = 8.0 + let hRect = CGRect( + x: item.point.x - hSize / 2, + y: item.point.y - hSize / 2, + width: hSize, + height: hSize + ) + context.setFillColor((isSpecial ? NSColor.systemYellow : NSColor.white).cgColor) + context.fillEllipse(in: hRect) + context.setStrokeColor(NSColor.controlAccentColor.cgColor) + context.setLineWidth(1.5) + context.strokeEllipse(in: hRect) + } + context.restoreGState() + } + // 5. Draw pixel loupe when hovering if case .hovering = state, let mousePos = mousePosition { drawLoupe(at: mousePos, context: context) @@ -309,6 +391,56 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { // MARK: - Selection Rendering Helpers + private func drawBeautifyPreview(for rect: CGRect, context: CGContext) { + let pad: CGFloat = 18.0 + let outerRect = rect.insetBy(dx: -pad, dy: -pad) + context.saveGState() + let framePath = CGMutablePath() + framePath.addRoundedRect(in: outerRect, cornerWidth: 14, cornerHeight: 14) + framePath.addRoundedRect(in: rect, cornerWidth: 8, cornerHeight: 8) + context.addPath(framePath) + context.clip(using: .evenOdd) + + let colors = BeautifyConfig.BackgroundStyle.Preset.ocean.colors.map { $0.cgColor } as CFArray + let colorSpace = context.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)! + if let gradient = CGGradient(colorsSpace: colorSpace, colors: colors, locations: [0.0, 1.0]) { + context.drawLinearGradient( + gradient, + start: CGPoint(x: outerRect.minX, y: outerRect.minY), + end: CGPoint(x: outerRect.maxX, y: outerRect.maxY), + options: [] + ) + } + context.restoreGState() + } + + /// Draws each translated segment as a strip just below its original + /// text inside the selection — in-image bilingual view; the original + /// screenshot stays fully visible. Only while the translate tab is active. + private func drawBilingualOverlay(for selRect: CGRect) { + guard resultsModel.isVisible, + resultsModel.activeTab == .translate, + !resultsModel.segments.isEmpty else { return } + + NSGraphicsContext.saveGraphicsState() + defer { NSGraphicsContext.restoreGraphicsState() } + NSBezierPath(rect: selRect).addClip() + for segment in resultsModel.segments { + guard let strip = BilingualOverlayLayout.translationStrip(for: segment, in: selRect.size) else { continue } + let attr = NSAttributedString(string: segment.translated, attributes: [ + .font: NSFont.systemFont(ofSize: strip.fontSize, weight: .medium), + .foregroundColor: NSColor.white + ]) + let frame = strip.frame.offsetBy(dx: selRect.minX, dy: selRect.minY) + NSColor.black.withAlphaComponent(0.78).setFill() + NSBezierPath(roundedRect: frame, xRadius: 3, yRadius: 3).fill() + attr.draw( + with: strip.textFrame.offsetBy(dx: selRect.minX, dy: selRect.minY), + options: [.usesLineFragmentOrigin, .usesFontLeading] + ) + } + } + private func drawSelectionBox(_ rect: CGRect, context: CGContext) { context.saveGState() context.setStrokeColor(NSColor.controlAccentColor.cgColor) @@ -522,9 +654,23 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { case .arrow(let a): context.setStrokeColor(a.style.color.nsColor.cgColor) context.setLineWidth(a.style.lineWidth) - context.move(to: a.startPoint) - context.addLine(to: a.endPoint) - context.strokePath() + let angle: CGFloat + if let cp = a.controlPoint { + context.move(to: a.startPoint) + context.addQuadCurve(to: a.endPoint, control: cp) + context.strokePath() + let dx = a.endPoint.x - cp.x + let dy = a.endPoint.y - cp.y + angle = atan2(dy, dx) + } else { + context.move(to: a.startPoint) + context.addLine(to: a.endPoint) + context.strokePath() + let dx = a.endPoint.x - a.startPoint.x + let dy = a.endPoint.y - a.startPoint.y + angle = atan2(dy, dx) + } + drawArrowHead(at: a.endPoint, angle: angle, length: max(10, a.style.lineWidth * 3.5), context: context, color: a.style.color.nsColor) case .freehand(let f): guard f.points.count >= 2 else { break } @@ -546,22 +692,32 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { (t.content as NSString).draw(in: t.bounds, withAttributes: attrs) case .mosaic(let m): - context.setFillColor(NSColor.gray.withAlphaComponent(0.6).cgColor) - context.fill(m.rect) + drawMosaicAnnotation(m, context: context) case .highlight(let h): context.setFillColor(h.color.nsColor.withAlphaComponent(h.opacity).cgColor) context.fill(h.rect) case .numberLabel(let n): + let circleRect = n.circleRect + if let lead = n.leadPoint { + context.setStrokeColor(n.color.nsColor.cgColor) + context.setLineWidth(2.0) + context.move(to: CGPoint(x: circleRect.midX, y: circleRect.midY)) + context.addLine(to: lead) + context.strokePath() + + context.setFillColor(n.color.nsColor.cgColor) + context.fillEllipse(in: CGRect(x: lead.x - 3.5, y: lead.y - 3.5, width: 7, height: 7)) + } context.setFillColor(n.color.nsColor.cgColor) - context.fillEllipse(in: n.bounds) + context.fillEllipse(in: circleRect) let numStr = "\(n.number)" let font = NSFont.boldSystemFont(ofSize: n.size * 0.6) let strSize = measureText(numStr, font: font) drawText( numStr, - at: CGPoint(x: n.bounds.midX - strSize.width / 2, y: n.bounds.midY - strSize.height / 2), + at: CGPoint(x: circleRect.midX - strSize.width / 2, y: circleRect.midY - strSize.height / 2), font: font, color: .white, in: context @@ -573,6 +729,26 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { case .calloutBubble(let cb): context.setFillColor(cb.bubbleColor.nsColor.cgColor) + if !cb.rect.contains(cb.targetPoint) { + let center = CGPoint(x: cb.rect.midX, y: cb.rect.midY) + let dx = cb.targetPoint.x - center.x + let dy = cb.targetPoint.y - center.y + let dist = max(1, hypot(dx, dy)) + let perpX = -dy / dist + let perpY = dx / dist + let tailHalfWidth = min(10.0, min(cb.rect.width, cb.rect.height) * 0.22) + + let base1 = CGPoint(x: center.x + perpX * tailHalfWidth, y: center.y + perpY * tailHalfWidth) + let base2 = CGPoint(x: center.x - perpX * tailHalfWidth, y: center.y - perpY * tailHalfWidth) + + let tailPath = CGMutablePath() + tailPath.move(to: base1) + tailPath.addLine(to: cb.targetPoint) + tailPath.addLine(to: base2) + tailPath.closeSubpath() + context.addPath(tailPath) + context.fillPath() + } let path = CGPath(roundedRect: cb.rect, cornerWidth: 8, cornerHeight: 8, transform: nil) context.addPath(path) context.fillPath() @@ -585,6 +761,93 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { context.restoreGState() } + private func drawMosaicAnnotation(_ mosaic: MosaicAnnotation, context: CGContext) { + let rect = mosaic.rect + guard rect.width > 1, rect.height > 1 else { return } + + let scale = displaySnapshot.scaleFactor + let imageBounds = CGRect( + x: 0, + y: 0, + width: CGFloat(displaySnapshot.image.width), + height: CGFloat(displaySnapshot.image.height) + ) + let pixelRect = CGRect( + x: rect.origin.x * scale, + y: rect.origin.y * scale, + width: rect.width * scale, + height: rect.height * scale + ).intersection(imageBounds) + + guard pixelRect.width > 1, pixelRect.height > 1 else { + context.setFillColor(NSColor.gray.withAlphaComponent(0.6).cgColor) + context.fill(rect) + return + } + + let pixelBlockSize = max(2, Int(round(CGFloat(mosaic.blockSize) * scale))) + let cacheKey = MosaicCache.shared.key( + rect: pixelRect, + blockSize: pixelBlockSize, + imageID: ObjectIdentifier(displaySnapshot.image) + ) + + let pixelatedCG: CGImage? + if let cached = MosaicCache.shared.image(for: cacheKey) { + pixelatedCG = cached + } else if let cropped = displaySnapshot.image.cropping(to: pixelRect) { + let ciImage = CIImage(cgImage: cropped) + if let filter = CIFilter(name: "CIPixellate") { + filter.setValue(ciImage, forKey: kCIInputImageKey) + filter.setValue( + CIVector(x: ciImage.extent.midX, y: ciImage.extent.midY), + forKey: kCIInputCenterKey + ) + filter.setValue(NSNumber(value: pixelBlockSize), forKey: kCIInputScaleKey) + if let output = filter.outputImage { + let croppedOutput = output.cropped(to: ciImage.extent) + let generated = Self.sharedCIContext.createCGImage(croppedOutput, from: ciImage.extent) + if let generated { + MosaicCache.shared.setImage(generated, for: cacheKey) + } + pixelatedCG = generated + } else { + pixelatedCG = nil + } + } else { + pixelatedCG = nil + } + } else { + pixelatedCG = nil + } + + if let pixelatedCG { + context.saveGState() + context.translateBy(x: rect.minX, y: rect.maxY) + context.scaleBy(x: 1.0, y: -1.0) + context.draw(pixelatedCG, in: CGRect(origin: .zero, size: rect.size)) + context.restoreGState() + } else { + context.setFillColor(NSColor.gray.withAlphaComponent(0.6).cgColor) + context.fill(rect) + } + } + + private func drawArrowHead(at point: CGPoint, angle: CGFloat, length: CGFloat, context: CGContext, color: NSColor) { + let arrowAngle: CGFloat = .pi / 6.0 + let p1 = CGPoint(x: point.x - length * cos(angle - arrowAngle), y: point.y - length * sin(angle - arrowAngle)) + let p2 = CGPoint(x: point.x - length * cos(angle + arrowAngle), y: point.y - length * sin(angle + arrowAngle)) + + context.saveGState() + context.setFillColor(color.cgColor) + context.move(to: point) + context.addLine(to: p1) + context.addLine(to: p2) + context.closePath() + context.fillPath() + context.restoreGState() + } + // MARK: - Mouse Interaction override func scrollWheel(with event: NSEvent) { @@ -627,48 +890,120 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { } override func rightMouseDown(with event: NSEvent) { + commitTextInput() dismissToolbar() delegate?.selectionOverlayDidCancel() } + private func hitTestAnnotation(at point: CGPoint) -> Int? { + committedAnnotations.indices.reversed().first { idx in + committedAnnotations[idx].hitTest(point: point, tolerance: 8.0) + } + } + override func mouseDown(with event: NSEvent) { let p = convert(event.locationInWindow, from: nil) - // 1. Double click: immediately confirm capture for target window/selection - if event.clickCount >= 2 { - handleDoubleClick(at: p) - return + if isEditingText { + commitTextInput() } - if let tool = activeAnnotationTool, case .selected(let selRect) = state, selRect.contains(p) { - // Start drawing annotation within selected region - dismissToolbar() - freehandPoints = [p] - state = .drawingAnnotation(tool: tool, selectionRect: selRect, start: p, current: p) - updateDraftAnnotation(start: p, current: p, tool: tool) - needsDisplay = true + // 1. Double click: re-edit text annotation if hit, otherwise confirm capture + if event.clickCount >= 2 { + if case .selected = state, + let hitIdx = hitTestAnnotation(at: p), + case .text = committedAnnotations[hitIdx] { + beginEditingTextAnnotation(at: hitIdx) + updateCursor(at: p) + return + } + handleDoubleClick(at: p) return } switch state { case .hovering: - dismissToolbar() + hideToolbar() + selectedAnnotationIndex = nil state = .creatingSelection(start: p, current: p) case .selected(let selRect): - dismissToolbar() + // 1. Check control handles of currently selected annotation + if let selIdx = selectedAnnotationIndex, + selIdx < committedAnnotations.count, + let annoHandle = committedAnnotations[selIdx].hitTestHandle(at: p, padding: 4, tolerance: 10) { + hideToolbar() + state = .resizingAnnotation( + index: selIdx, + selectionRect: selRect, + handle: annoHandle, + originalAnnotations: committedAnnotations + ) + updateCursor(at: p) + needsDisplay = true + return + } + + // 2. Check selection box resize handles if let handle = hitTestHandle(at: p, in: selRect) { + hideToolbar() + selectedAnnotationIndex = nil state = .resizing(handle: handle, anchorRect: selRect, startPoint: p, currentRect: selRect) - } else if selRect.contains(p) { - state = .moving(anchorRect: selRect, startPoint: p, currentRect: selRect) + updateCursor(at: p) + needsDisplay = true + return + } + + // 3. Check hit on any existing annotation (allows selecting/dragging existing annotations even when a tool is active) + // Single click selects and drags; double click (handled above) re-edits text + if let hitIdx = hitTestAnnotation(at: p) { + selectedAnnotationIndex = hitIdx + hideToolbar() + state = .movingAnnotation( + index: hitIdx, + selectionRect: selRect, + startPoint: p, + originalAnno: committedAnnotations[hitIdx], + originalAnnotations: committedAnnotations + ) + updateCursor(at: p) + needsDisplay = true + return + } + + // 4. Inside selection rect: start drawing with active tool or move selection box + if selRect.contains(p) { + if let tool = activeAnnotationTool { + if tool == .text { + beginTextInput(at: p, editingIndex: nil) + updateCursor(at: p) + return + } + selectedAnnotationIndex = nil + hideToolbar() + freehandPoints = [p] + state = .drawingAnnotation(tool: tool, selectionRect: selRect, start: p, current: p) + updateDraftAnnotation(start: p, current: p, tool: tool) + } else { + selectedAnnotationIndex = nil + hideToolbar() + state = .moving(anchorRect: selRect, startPoint: p, currentRect: selRect) + } } else { - // Click outside: reset selection back to creating selection + // Click outside the selection: only a real drag starts a new + // selection — a plain click must leave the selection untouched. + // The results drawer is hidden later, on confirmed drag/new + // selection, so a stray click doesn't discard OCR results. + selectionBeforeOutsideClick = selRect + selectedAnnotationIndex = nil + hideToolbar() state = .creatingSelection(start: p, current: p) } default: break } + updateCursor(at: p) needsDisplay = true } @@ -679,18 +1014,40 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { switch state { case .creatingSelection(let start, _): state = .creatingSelection(start: start, current: p) + // A real drag is starting: retire the previous selection's drawer. + if resultsModel.isVisible, hypot(p.x - start.x, p.y - start.y) > 4 { + hideResultsPanel() + } case .resizing(let handle, let anchor, let start, _): let dx = p.x - start.x let dy = p.y - start.y let newRect = resizeRect(anchor, handle: handle, dx: dx, dy: dy) state = .resizing(handle: handle, anchorRect: anchor, startPoint: start, currentRect: newRect) + layoutResultsPanel(for: newRect) case .moving(let anchor, let start, _): let dx = p.x - start.x let dy = p.y - start.y let moved = anchor.offsetBy(dx: dx, dy: dy).intersection(bounds) state = .moving(anchorRect: anchor, startPoint: start, currentRect: moved) + layoutResultsPanel(for: moved) + + case .movingAnnotation(let index, _, let start, let originalAnno, _): + let dx = p.x - start.x + let dy = p.y - start.y + if index < committedAnnotations.count { + committedAnnotations[index] = originalAnno.offsetBy(dx: dx, dy: dy) + } + + case .resizingAnnotation(let index, _, let handle, _): + if index < committedAnnotations.count { + // ⌘ held: lock the current aspect ratio + let options = HandleDragOptions( + preserveAspectRatio: NSEvent.modifierFlags.contains(.command) + ) + committedAnnotations[index] = committedAnnotations[index].applyingHandleDrag(handle, to: p, options: options) + } case .drawingAnnotation(let tool, let selRect, let start, _): freehandPoints.append(p) @@ -700,15 +1057,26 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { default: break } + updateCursor(at: p) needsDisplay = true } override func mouseUp(with event: NSEvent) { + let p = convert(event.locationInWindow, from: nil) + switch state { case .creatingSelection(let start, let current): let rect = normalizedRect(from: start, to: current) if rect.width >= 5 && rect.height >= 5 { + selectionBeforeOutsideClick = nil + hideResultsPanel() commitSelection(rect) + performAutomaticSelectionAction(in: rect) + } else if let previous = selectionBeforeOutsideClick { + // Plain click outside the selection: no reaction, keep the + // selection and its toolbar exactly as they were. + selectionBeforeOutsideClick = nil + commitSelection(previous) } else if !isFreeSelectionLocked { if let win = candidateWindowsUnderMouse.first { let local = CoordinateMapper.quartzToDisplayLocal( @@ -729,19 +1097,31 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { case .moving(_, _, let currentRect): commitSelection(currentRect) + case .movingAnnotation(_, let selRect, _, _, let originalAnnotations): + recordAnnotationHistory( + description: "Move annotation", + from: originalAnnotations, + to: committedAnnotations + ) + commitSelection(selRect) + + case .resizingAnnotation(_, let selRect, _, let originalAnnotations): + recordAnnotationHistory( + description: "Resize annotation", + from: originalAnnotations, + to: committedAnnotations + ) + commitSelection(selRect) + case .drawingAnnotation(let tool, let selRect, _, _): if let draft = currentDraftAnnotation { + let oldSnapshot = committedAnnotations committedAnnotations.append(draft) - editHistory.record( + selectedAnnotationIndex = committedAnnotations.count - 1 + recordAnnotationHistory( description: "Add \(tool.rawValue)", - undo: { [weak self] in - _ = self?.committedAnnotations.popLast() - self?.needsDisplay = true - }, - redo: { [weak self] in - self?.committedAnnotations.append(draft) - self?.needsDisplay = true - } + from: oldSnapshot, + to: committedAnnotations ) } currentDraftAnnotation = nil @@ -751,10 +1131,13 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { default: break } + updateCursor(at: p) needsDisplay = true } private func handleDoubleClick(at point: CGPoint) { + commitTextInput() + let targetRect: CGRect switch state { case .selected(let rect): @@ -785,11 +1168,12 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { displayQuartzBounds: displaySnapshot.quartzBounds ) + let exportAnnotations = transformedAnnotationsForExport(in: targetRect) dismissToolbar() delegate?.selectionOverlay( didConfirmSelection: quartzSel, action: .copy, - annotations: committedAnnotations, + annotations: exportAnnotations, on: displaySnapshot ) } @@ -797,48 +1181,297 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { private func commitSelection(_ rect: CGRect) { state = .selected(rect: rect) showToolbar(for: rect) + // Drawer and selection form one interaction unit: keep them attached + if resultsModel.isVisible { + resultsPanelHost?.isHidden = false + layoutResultsPanel(for: rect) + } } private func updateDraftAnnotation(start: CGPoint, current: CGPoint, tool: SelectionInPlaceToolbar.AnnotationTool) { let style = StrokeStyle(color: CodableColor(Color(nsColor: annotationColor)), lineWidth: annotationLineWidth) - let rect = normalizedRect(from: start, to: current) + var rect = normalizedRect(from: start, to: current) + + // ⌘ held: constrain rectangle/ellipse to a square / circle (1:1) + if NSEvent.modifierFlags.contains(.command), tool == .rectangle || tool == .ellipse { + rect = squareRect(from: start, to: current) + } switch tool { case .rectangle: currentDraftAnnotation = .rectangle(RectangleAnnotation(rect: rect, style: style)) case .ellipse: currentDraftAnnotation = .ellipse(EllipseAnnotation(rect: rect, style: style)) + case .line: + currentDraftAnnotation = .line(LineAnnotation(startPoint: start, endPoint: current, style: style)) case .arrow: currentDraftAnnotation = .arrow(ArrowAnnotation(startPoint: start, endPoint: current, style: style)) case .pen: currentDraftAnnotation = .freehand(FreehandAnnotation(points: freehandPoints.isEmpty ? [start, current] : freehandPoints, style: style)) case .text: - currentDraftAnnotation = .text(TextAnnotation(position: rect.origin, content: "Text")) + break + case .numberLabel: + let nextNum = committedAnnotations.filter { + if case .numberLabel = $0 { return true } else { return false } + }.count + 1 + currentDraftAnnotation = .numberLabel( + NumberLabelAnnotation( + position: start, + number: nextNum, + size: 26, + color: CodableColor(Color(nsColor: annotationColor)), + leadPoint: (hypot(current.x - start.x, current.y - start.y) > 4) ? current : nil + ) + ) + case .highlight: + currentDraftAnnotation = .highlight( + HighlightAnnotation( + rect: rect, + color: CodableColor(Color(nsColor: annotationColor)), + opacity: 0.4 + ) + ) case .mosaic: - currentDraftAnnotation = .mosaic(MosaicAnnotation(rect: rect)) + currentDraftAnnotation = .mosaic(MosaicAnnotation(rect: rect, blockSize: mosaicBlockSize)) case .privacyRedaction: currentDraftAnnotation = .privacyRedaction(PrivacyRedactionAnnotation(rect: rect)) } } + // MARK: - In-Place Text Editing + + private func beginEditingTextAnnotation(at index: Int) { + guard index >= 0, index < committedAnnotations.count else { return } + if case .text(let textAnno) = committedAnnotations[index] { + selectedAnnotationIndex = index + beginTextInput(at: textAnno.position, editingIndex: index, initialText: textAnno.content, fontSize: textAnno.style.fontSize, color: textAnno.style.color.nsColor) + } + } + + private func beginTextInput( + at point: CGPoint, + editingIndex: Int?, + initialText: String = "", + fontSize: CGFloat? = nil, + color: NSColor? = nil + ) { + if isEditingText { + commitTextInput() + } + + let resolvedFontSize = fontSize ?? max(14.0, annotationLineWidth * 5.0) + let resolvedColor = color ?? annotationColor + let font = NSFont.systemFont(ofSize: resolvedFontSize) + + editingAnnotationIndex = editingIndex + textInputPosition = point + + let tf = NSTextField(frame: .zero) + tf.isBordered = false + tf.drawsBackground = true + tf.backgroundColor = NSColor.textBackgroundColor.withAlphaComponent(0.9) + tf.focusRingType = .none + tf.font = font + tf.textColor = resolvedColor + tf.placeholderString = String(localized: "preview.enter.text") + tf.stringValue = initialText + tf.delegate = self + tf.wantsLayer = true + tf.layer?.cornerRadius = 4 + tf.layer?.borderWidth = 1.5 + tf.layer?.borderColor = NSColor.controlAccentColor.cgColor + + addSubview(tf) + activeTextField = tf + updateActiveTextFieldFrame() + window?.makeFirstResponder(tf) + if let editor = tf.currentEditor() { + editor.selectedRange = NSRange(location: tf.stringValue.count, length: 0) + } + needsDisplay = true + } + + private func updateActiveTextFieldFrame() { + guard let tf = activeTextField, let pos = textInputPosition else { return } + let font = tf.font ?? NSFont.systemFont(ofSize: 16) + let sample = tf.stringValue.isEmpty ? (tf.placeholderString ?? "Text") : tf.stringValue + let measured = measureText(sample, font: font) + let width = max(90, measured.width + 20) + let height = max(24, measured.height + 8) + let clampedX = max(4, min(bounds.maxX - width - 4, pos.x)) + let clampedY = max(4, min(bounds.maxY - height - 4, pos.y)) + tf.frame = CGRect(x: clampedX, y: clampedY, width: width, height: height) + } + + func commitTextInput() { + guard let tf = activeTextField else { return } + let rawText = tf.stringValue.trimmingCharacters(in: .whitespacesAndNewlines) + let editIdx = editingAnnotationIndex + let pos = textInputPosition + let fontSize = tf.font?.pointSize ?? max(14.0, annotationLineWidth * 5.0) + let textColor = tf.textColor ?? annotationColor + + tf.delegate = nil + tf.removeFromSuperview() + activeTextField = nil + editingAnnotationIndex = nil + textInputPosition = nil + window?.makeFirstResponder(self) + + if let editIdx, editIdx >= 0, editIdx < committedAnnotations.count { + if case .text(var textAnno) = committedAnnotations[editIdx] { + let oldSnapshot = committedAnnotations + if rawText.isEmpty { + committedAnnotations.remove(at: editIdx) + selectedAnnotationIndex = nil + recordAnnotationHistory(description: "Delete text", from: oldSnapshot, to: committedAnnotations) + } else if textAnno.content != rawText { + textAnno.content = rawText + committedAnnotations[editIdx] = .text(textAnno) + recordAnnotationHistory(description: "Edit text", from: oldSnapshot, to: committedAnnotations) + } + } + } else if let pos, !rawText.isEmpty { + let oldSnapshot = committedAnnotations + let style = TextStyle( + color: CodableColor(Color(nsColor: textColor)), + fontSize: fontSize, + fontName: ".AppleSystemUIFont" + ) + let newAnno = Annotation.text(TextAnnotation(position: pos, content: rawText, style: style)) + committedAnnotations.append(newAnno) + selectedAnnotationIndex = committedAnnotations.count - 1 + recordAnnotationHistory(description: "Add text", from: oldSnapshot, to: committedAnnotations) + } + + needsDisplay = true + } + + func cancelTextInput() { + guard let tf = activeTextField else { return } + tf.delegate = nil + tf.removeFromSuperview() + activeTextField = nil + editingAnnotationIndex = nil + textInputPosition = nil + window?.makeFirstResponder(self) + needsDisplay = true + } + + func controlTextDidChange(_ obj: Notification) { + updateActiveTextFieldFrame() + } + + func controlTextDidEndEditing(_ obj: Notification) { + if activeTextField != nil { + commitTextInput() + } + } + + func control(_ control: NSControl, textView: NSTextView, doCommandBy commandSelector: Selector) -> Bool { + if commandSelector == #selector(NSResponder.insertNewline(_:)) { + commitTextInput() + return true + } + if commandSelector == #selector(NSResponder.cancelOperation(_:)) { + cancelTextInput() + return true + } + return false + } + + // MARK: - Undo/Redo History Helpers + + private func recordAnnotationHistory( + description: String, + from oldAnnotations: [Annotation], + to newAnnotations: [Annotation] + ) { + guard oldAnnotations != newAnnotations else { return } + editHistory.record( + description: description, + undo: { [weak self] in + guard let self else { return } + self.committedAnnotations = oldAnnotations + if let sel = self.selectedAnnotationIndex, sel >= self.committedAnnotations.count { + self.selectedAnnotationIndex = nil + } + self.syncToolbarUndoState() + self.needsDisplay = true + }, + redo: { [weak self] in + guard let self else { return } + self.committedAnnotations = newAnnotations + if let sel = self.selectedAnnotationIndex, sel >= self.committedAnnotations.count { + self.selectedAnnotationIndex = nil + } + self.syncToolbarUndoState() + self.needsDisplay = true + } + ) + syncToolbarUndoState() + } + + private func syncToolbarUndoState() { + toolbar?.state.canUndo = editHistory.canUndo + toolbar?.state.canRedo = editHistory.canRedo + } + + private func transformedAnnotationsForExport(in selectionRect: CGRect) -> [Annotation] { + let originX = selectionRect.origin.x + let originY = selectionRect.origin.y + let scale = displaySnapshot.scaleFactor + return committedAnnotations.map { anno in + anno.offsetBy(dx: -originX, dy: -originY).scaled(by: scale) + } + } + // MARK: - Toolbar & Actions private func showToolbar(for selectionRect: CGRect) { - dismissToolbar() - let bar = SelectionInPlaceToolbar(frame: .zero) - bar.delegate = self - let targetFrame = SelectionInPlaceToolbar.preferredFrame(for: selectionRect, in: bounds) + let bar: SelectionInPlaceToolbar + if let existing = toolbar { + bar = existing + } else { + let newBar = SelectionInPlaceToolbar(frame: .zero) + newBar.delegate = self + addSubview(newBar) + self.toolbar = newBar + bar = newBar + } + syncToolbarUndoState() + bar.state.isBeautifyEnabled = isBeautifyEnabled + let targetFrame = SelectionInPlaceToolbar.preferredFrame( + for: selectionRect, + in: bounds, + toolbarSize: bar.preferredSize + ) bar.frame = targetFrame - addSubview(bar) - self.toolbar = bar + bar.isHidden = false + } + + private func hideToolbar() { + toolbar?.isHidden = true } private func dismissToolbar() { + hideResultsPanel() toolbar?.removeFromSuperview() toolbar = nil } + func toolbarDidUpdateLayout() { + guard case .selected(let rect) = state, let bar = toolbar, !bar.isHidden else { return } + bar.frame = SelectionInPlaceToolbar.preferredFrame( + for: rect, + in: bounds, + toolbarSize: bar.preferredSize + ) + } + func toolbarDidSelectAction(_ action: SelectionInPlaceToolbar.Action) { + commitTextInput() + guard case .selected(let rect) = state else { if action == .cancel { delegate?.selectionOverlayDidCancel() @@ -851,48 +1484,459 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { return } + if action == .beautify { + isBeautifyEnabled.toggle() + toolbar?.state.isBeautifyEnabled = isBeautifyEnabled + needsDisplay = true + return + } + + if action == .smartMosaic { + runSmartRedaction(in: rect) + return + } + + // OCR / Translate: expand the in-place results drawer, never leave this window + if action == .ocr { + runInPlaceAnalysis(tab: .ocr, in: rect) + return + } + if action == .translate { + runInPlaceAnalysis(tab: .translate, in: rect) + return + } + // Convert DisplayLocal selection rect to Quartz global rect let quartzSel = CoordinateMapper.displayLocalToQuartz( DisplayLocalRect(rect), displayQuartzBounds: displaySnapshot.quartzBounds ) + let exportAnnotations = transformedAnnotationsForExport(in: rect) + delegate?.selectionOverlay( didConfirmSelection: quartzSel, action: action, - annotations: committedAnnotations, + annotations: exportAnnotations, on: displaySnapshot ) } func toolbarDidChangeTool(_ tool: SelectionInPlaceToolbar.AnnotationTool?) { + commitTextInput() activeAnnotationTool = tool } func toolbarDidChangeColor(_ color: NSColor) { annotationColor = color + if let tf = activeTextField { + tf.textColor = color + } + if let idx = selectedAnnotationIndex, idx < committedAnnotations.count { + let oldSnapshot = committedAnnotations + committedAnnotations[idx] = committedAnnotations[idx].updatingColor(CodableColor(Color(nsColor: color))) + recordAnnotationHistory(description: "Change color", from: oldSnapshot, to: committedAnnotations) + needsDisplay = true + } + } + + // MARK: - Smart Privacy Mosaic + + /// Detects sensitive privacy regions inside the selection and adds mosaic + /// annotations in one undoable step. Requires a configured VLM API key. + private func runSmartRedaction(in selectionRect: CGRect) { + + let scale = displaySnapshot.scaleFactor + let imageBounds = CGRect( + x: 0, + y: 0, + width: CGFloat(displaySnapshot.image.width), + height: CGFloat(displaySnapshot.image.height) + ) + let pixelRect = CGRect( + x: selectionRect.origin.x * scale, + y: selectionRect.origin.y * scale, + width: selectionRect.width * scale, + height: selectionRect.height * scale + ).intersection(imageBounds) + guard pixelRect.width > 1, pixelRect.height > 1, + let cropped = displaySnapshot.image.cropping(to: pixelRect) else { + showStatusMessage(String(localized: "smartMosaic.error.failed"), isError: true) + return + } + + showStatusMessage(String(localized: "smartMosaic.detecting"), isError: false) + + Task { @MainActor [weak self] in + guard let self else { return } + do { + let regions = try await SmartPrivacyRedactionService.shared.detectPrivacyRegions(in: cropped) + guard !regions.isEmpty else { + self.showStatusMessage(String(localized: "smartMosaic.noneFound"), isError: false) + return + } + + // Convert crop-pixel rects back to display-local coordinates + let localRects = regions.map { region -> CGRect in + CGRect( + x: selectionRect.origin.x + region.origin.x / scale, + y: selectionRect.origin.y + region.origin.y / scale, + width: region.width / scale, + height: region.height / scale + ) + } + + let oldSnapshot = self.committedAnnotations + for rect in localRects { + self.committedAnnotations.append( + .mosaic(MosaicAnnotation(rect: rect, blockSize: self.mosaicBlockSize)) + ) + } + self.recordAnnotationHistory( + description: "Smart redaction", + from: oldSnapshot, + to: self.committedAnnotations + ) + self.needsDisplay = true + self.showStatusMessage( + String(format: String(localized: "smartMosaic.redacted"), localRects.count), + isError: false + ) + } catch { + self.showStatusMessage(String(localized: "smartMosaic.error.failed"), isError: true) + } + } + } + + /// Transient status message near the toolbar + private func showStatusMessage(_ message: String, isError: Bool) { + statusHideWorkItem?.cancel() + + let label = statusLabel ?? { + let field = NSTextField(labelWithString: "") + field.wantsLayer = true + field.layer?.cornerRadius = 6 + field.layer?.backgroundColor = NSColor.black.withAlphaComponent(0.78).cgColor + field.textColor = .white + field.font = .systemFont(ofSize: 12, weight: .medium) + field.alignment = .center + addSubview(field) + statusLabel = field + return field + }() + + label.stringValue = message + label.textColor = isError ? NSColor.systemOrange : NSColor.white + label.sizeToFit() + let size = NSSize(width: label.frame.width + 20, height: label.frame.height + 10) + let anchor = toolbar?.frame ?? NSRect(x: bounds.midX, y: 40, width: 0, height: 0) + label.frame = NSRect( + x: max(8, anchor.midX - size.width / 2), + y: anchor.minY + anchor.height + 8, + width: size.width, + height: size.height + ) + label.isHidden = false + + let work = DispatchWorkItem { [weak self] in + self?.statusLabel?.isHidden = true + } + statusHideWorkItem = work + DispatchQueue.main.asyncAfter(deadline: .now() + 2.8, execute: work) + } + + // MARK: - In-Place Results Drawer (OCR / Translation) + + /// Runs the mode's automatic action after a fresh selection commit. + /// A plain click (selection restore) never reaches here. + private func performAutomaticSelectionAction(in selectionRect: CGRect) { + guard let action = automaticActionAfterSelection else { return } + switch action { + case .translate: + runInPlaceAnalysis(tab: .translate, in: selectionRect) + case .ocr: + runInPlaceAnalysis(tab: .ocr, in: selectionRect) + default: + toolbarDidSelectAction(action) + } + } + + /// Runs OCR or translation on the selection and shows results in the + /// embedded right-side drawer. The capture window stays open. + private func runInPlaceAnalysis(tab: InPlaceResultsModel.Tab, in selectionRect: CGRect) { + // Second click on the same tab closes the drawer (PRD toggle behaviour) + if resultsModel.isVisible, resultsModel.activeTab == tab, !resultsModel.isLoading { + hideResultsPanel() + return + } + + guard let cropped = cropSelection(selectionRect) else { + showStatusMessage(String(localized: "error.ocr.failed"), isError: true) + return + } + + showResultsPanel(for: selectionRect, tab: tab) + resultsModel.isLoading = true + resultsModel.ocrLines = [] + resultsModel.segments = [] + resultsModel.statusMessage = nil + + // Cancel any in-flight run so a slow old result can't overwrite a new one + inPlaceAnalysisTask?.cancel() + inPlaceAnalysisTask = Task { @MainActor [weak self] in + guard let self, !Task.isCancelled else { return } + defer { + self.resultsModel.isLoading = false + self.layoutResultsPanel(for: selectionRect) + self.needsDisplay = true + } + do { + switch tab { + case .ocr: + let result = try await OCRService.shared.recognize( + cropped, + // 识别语言跟随设置的源语言;auto / 不支持语言回退自动检测 + languages: Set( + [AppSettings.shared.translationSourceLanguage] + .compactMap { OCREngine.RecognitionLanguage(translationLanguage: $0) } + ) + ) + if Task.isCancelled { return } + let lines = result.observations + .sorted { $0.boundingBox.minY < $1.boundingBox.minY } + .map(\.text) + self.resultsModel.ocrLines = lines + if lines.isEmpty { + self.resultsModel.statusMessage = String(localized: "error.ocr.no.text") + } + case .translate: + let pipeline = ImageTranslationPipeline() + let context = PipelineContext() + // Stage-split so failures are attributed correctly: the + // drawer reads VLM errors as recognition, not translation. + let analysis: ScreenAnalysisResult + do { + analysis = try await pipeline.analyze(image: cropped, context: context) + } catch { + self.resultsModel.statusMessage = String( + format: String(localized: "error.pipeline.ocrFailed"), + Self.pipelineErrorMessage(error) + ) + return + } + let filtered = analysis.filteredForTranslation() + guard !filtered.segments.isEmpty else { + self.resultsModel.statusMessage = String(localized: "error.ocr.no.text") + return + } + let config = ImageTranslationPipelineConfig.fromAppSettings(scene: nil) + do { + let segments = try await pipeline.translate( + analysisResult: filtered, + config: config, + context: context + ) + if Task.isCancelled { return } + self.resultsModel.segments = segments + } catch { + self.resultsModel.statusMessage = String( + format: String(localized: "error.pipeline.translateFailed"), + Self.pipelineErrorMessage(error) + ) + } + } + } catch { + let message = Self.pipelineErrorMessage(error) + self.resultsModel.statusMessage = message + } + } + } + + private static func pipelineErrorMessage(_ error: Error) -> String { + (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + } + + /// Crops the selection region from the frozen snapshot (pixel space). + private func cropSelection(_ selectionRect: CGRect) -> CGImage? { + let scale = displaySnapshot.scaleFactor + let imageBounds = CGRect( + x: 0, + y: 0, + width: CGFloat(displaySnapshot.image.width), + height: CGFloat(displaySnapshot.image.height) + ) + let pixelRect = CGRect( + x: selectionRect.origin.x * scale, + y: selectionRect.origin.y * scale, + width: selectionRect.width * scale, + height: selectionRect.height * scale + ).intersection(imageBounds) + guard pixelRect.width > 1, pixelRect.height > 1 else { return nil } + return displaySnapshot.image.cropping(to: pixelRect) + } + + private func showResultsPanel(for selectionRect: CGRect, tab: InPlaceResultsModel.Tab) { + if resultsPanelHost == nil { + resultsModel.onRerun = { [weak self] in + guard let self, case .selected(let rect) = self.state else { return } + self.runInPlaceAnalysis(tab: self.resultsModel.activeTab, in: rect) + } + resultsModel.onClose = { [weak self] in + self?.hideResultsPanel() + } + let host = NSHostingView(rootView: InPlaceResultsPanel(model: resultsModel)) + host.autoresizingMask = [] + addSubview(host) + resultsPanelHost = host + } + resultsModel.visualInvalidator = { [weak self] in self?.needsDisplay = true } + resultsModel.activeTab = tab + resultsModel.isVisible = true + resultsPanelHost?.isHidden = false + layoutResultsPanel(for: selectionRect) + } + + private func layoutResultsPanel(for selectionRect: CGRect) { + guard let host = resultsPanelHost, resultsModel.isVisible else { return } + host.frame = Self.resultsPanelFrame( + selectionRect: selectionRect, + bounds: bounds, + handleSize: handleSize + ) + } + + /// Frame for the merged results drawer, in flipped (Y-down) local coords. + /// Sits beside the selection's right edge with a gap that clears the + /// resize handles (whose hit areas reach `handleSize + 2` past the edge — + /// child views win hit testing, so an overlapping handle would swallow + /// its clicks), and never enters the toolbar zone below the selection + /// (toolbar top sits at maxY + 8). + static func resultsPanelFrame( + selectionRect: CGRect, + bounds: CGRect, + handleSize: CGFloat + ) -> CGRect { + let gap: CGFloat = handleSize + 6 + let width: CGFloat = 300 + let desiredHeight = max(220, min(selectionRect.height, bounds.maxY - 16)) + let x: CGFloat = { + let preferred = selectionRect.maxX + gap + if preferred + width <= bounds.maxX - 8 { return preferred } + // Left side preferred; when neither side fits (near-fullscreen + // selection) hug the screen's right edge. + let left = selectionRect.minX - width - gap + return left >= 8 ? left : bounds.maxX - width - 8 + }() + let y = max(8, min(selectionRect.maxY - desiredHeight, bounds.maxY - desiredHeight - 8)) + // Cap the height so the drawer bottom never crosses the toolbar top. + let toolbarZoneTop = selectionRect.maxY + 8 + let height = min(desiredHeight, max(40, toolbarZoneTop - y), bounds.maxY - 8 - y) + return NSRect(x: x, y: y, width: width, height: height) + } + + private func hideResultsPanel() { + inPlaceAnalysisTask?.cancel() + resultsModel.isVisible = false + resultsPanelHost?.isHidden = true + needsDisplay = true + } + + func toolbarDidChangeMosaicIntensity(_ blockSize: Int) { + mosaicBlockSize = blockSize + if let idx = selectedAnnotationIndex, idx < committedAnnotations.count, + case .mosaic(var mosaic) = committedAnnotations[idx] { + let oldSnapshot = committedAnnotations + mosaic.blockSize = blockSize + committedAnnotations[idx] = .mosaic(mosaic) + recordAnnotationHistory(description: "Change mosaic intensity", from: oldSnapshot, to: committedAnnotations) + needsDisplay = true + } } func toolbarDidChangeLineWidth(_ width: CGFloat) { annotationLineWidth = width + if let tf = activeTextField { + tf.font = NSFont.systemFont(ofSize: max(14.0, width * 5.0)) + updateActiveTextFieldFrame() + } + if let idx = selectedAnnotationIndex, idx < committedAnnotations.count { + let oldSnapshot = committedAnnotations + committedAnnotations[idx] = committedAnnotations[idx].updatingLineWidth(width) + recordAnnotationHistory(description: "Change line width", from: oldSnapshot, to: committedAnnotations) + needsDisplay = true + } } func toolbarDidRequestUndo() { + commitTextInput() editHistory.undo() + syncToolbarUndoState() } func toolbarDidRequestRedo() { + commitTextInput() editHistory.redo() + syncToolbarUndoState() } // MARK: - Keyboard Controls override func keyDown(with event: NSEvent) { + // While the results drawer owns keyboard focus, its controls (and the + // text view) keep their keys: only Escape falls through, closing the + // drawer instead of cancelling the whole capture session. + if resultsPanelHasKeyboardFocus { + if event.keyCode == 53 { + hideResultsPanel() + window?.makeFirstResponder(self) + return + } + super.keyDown(with: event) + return + } + + if isEditingText { + if event.keyCode == 53 { + cancelTextInput() + return + } + if event.keyCode == 36 || event.keyCode == 76 { + commitTextInput() + return + } + super.keyDown(with: event) + return + } + + // ⌘Z / ⌘⇧Z for Undo / Redo + if event.modifierFlags.contains(.command) && event.keyCode == 6 { + if event.modifierFlags.contains(.shift) { + toolbarDidRequestRedo() + } else { + toolbarDidRequestUndo() + } + return + } + switch event.keyCode { case 53: // Escape dismissToolbar() delegate?.selectionOverlayDidCancel() + case 51: // Backspace / Delete + if let idx = selectedAnnotationIndex, idx < committedAnnotations.count { + let oldSnapshot = committedAnnotations + committedAnnotations.remove(at: idx) + selectedAnnotationIndex = nil + recordAnnotationHistory(description: "Delete annotation", from: oldSnapshot, to: committedAnnotations) + needsDisplay = true + } + + case 1: // S -> Save + if event.modifierFlags.contains(.command) { + toolbarDidSelectAction(.save) + } + case 36: // Enter toolbarDidSelectAction(.copy) @@ -1010,18 +2054,99 @@ final class SelectionOverlayView: NSView, SelectionToolbarDelegate { return normalizedRect(from: CGPoint(x: minX, y: minY), to: CGPoint(x: maxX, y: maxY)) } + /// True when a control inside the results drawer (e.g. its text view or + /// the language menu) is the window's first responder. + private var resultsPanelHasKeyboardFocus: Bool { + guard let host = resultsPanelHost, !host.isHidden, + let responder = window?.firstResponder as? NSView else { return false } + var current: NSView? = responder + while let view = current { + if view === host { return true } + current = view.superview + } + return false + } + private func updateCursor(at point: CGPoint) { - if case .selected(let selRect) = state { + if let tf = activeTextField, !tf.isHidden, tf.frame.contains(point) { + NSCursor.iBeam.set() + return + } + + if let bar = toolbar, !bar.isHidden, bar.frame.contains(point) { + NSCursor.arrow.set() + return + } + + // The results drawer hosts its own cursor management (arrow/iBeam); + // never show selection cursors (crosshair/grab) on top of it. + if let panel = resultsPanelHost, !panel.isHidden, panel.frame.contains(point) { + NSCursor.arrow.set() + return + } + + switch state { + case .moving, .movingAnnotation: + NSCursor.closedHand.set() + return + + case .resizing(let handle, _, _, _): + handle.cursor().set() + return + + case .resizingAnnotation(_, _, let handle, _): + handle.cursor.set() + return + + case .drawingAnnotation(let tool, _, _, _): + tool.cursor.set() + return + + case .creatingSelection: + NSCursor.crosshair.set() + return + + case .selected(let selRect): + if let selIdx = selectedAnnotationIndex, + selIdx < committedAnnotations.count, + let annoHandle = committedAnnotations[selIdx].hitTestHandle(at: point, padding: 4, tolerance: 10) { + annoHandle.cursor.set() + return + } + if let handle = hitTestHandle(at: point, in: selRect) { handle.cursor().set() return } + + if let hitIdx = hitTestAnnotation(at: point) { + if activeAnnotationTool == .text, case .text = committedAnnotations[hitIdx] { + NSCursor.iBeam.set() + return + } + NSCursor.openHand.set() + return + } + if selRect.contains(point) { + if let tool = activeAnnotationTool { + tool.cursor.set() + return + } NSCursor.openHand.set() return } + + NSCursor.crosshair.set() + + case .hovering: + NSCursor.crosshair.set() } - NSCursor.crosshair.set() + } + + /// 1:1 rectangle from start to current point (⌘ constraint) + private func squareRect(from start: CGPoint, to end: CGPoint) -> CGRect { + GeometryConstraint.squareRect(from: start, to: end) } private func normalizedRect(from start: CGPoint, to end: CGPoint) -> CGRect { diff --git a/TransFrame/Features/History/HistoryView.swift b/TransFrame/Features/History/HistoryView.swift index 47308b0..72db998 100644 --- a/TransFrame/Features/History/HistoryView.swift +++ b/TransFrame/Features/History/HistoryView.swift @@ -13,6 +13,9 @@ struct HistoryView: View { // Search bar SearchBar(store: store) + // Category & Favorites Filter Bar + CategoryFilterBar(store: store) + Divider() // History list @@ -113,6 +116,65 @@ private struct SearchBar: View { } } +// MARK: - Category Filter Bar + +private struct CategoryFilterBar: View { + @ObservedObject var store: HistoryStore + + var body: some View { + HStack(spacing: 6) { + filterButton(title: String(localized: "history.filter.all"), category: nil) + + ForEach(HistoryCategory.allCases, id: \.self) { cat in + filterButton(title: cat.displayName, category: cat, icon: cat.systemImage) + } + + Spacer() + + Button { + store.onlyFavorites.toggle() + } label: { + HStack(spacing: 3) { + Image(systemName: store.onlyFavorites ? "star.fill" : "star") + .foregroundColor(store.onlyFavorites ? .yellow : .secondary) + Text(String(localized: "history.filter.starred")) + .font(.system(size: 11, weight: store.onlyFavorites ? .semibold : .regular)) + } + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(store.onlyFavorites ? Color.yellow.opacity(0.15) : Color.clear) + .cornerRadius(12) + } + .buttonStyle(.plain) + } + .padding(.horizontal, 12) + .padding(.bottom, 8) + .background(Color(nsColor: .controlBackgroundColor)) + } + + private func filterButton(title: String, category: HistoryCategory?, icon: String? = nil) -> some View { + let isSelected = store.selectedCategory == category + return Button { + store.selectedCategory = category + } label: { + HStack(spacing: 3) { + if let icon { + Image(systemName: icon) + .font(.system(size: 10)) + } + Text(title) + .font(.system(size: 11, weight: isSelected ? .semibold : .regular)) + } + .foregroundColor(isSelected ? .white : .primary) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(isSelected ? Color.accentColor : Color.secondary.opacity(0.12)) + .cornerRadius(12) + } + .buttonStyle(.plain) + } +} + // MARK: - Empty State /// View shown when no history entries exist. @@ -185,14 +247,27 @@ private struct HistoryEntryRow: View { var body: some View { VStack(alignment: .leading, spacing: 6) { - // Header with languages and timestamp - HStack { + // Header with category icon, description, star, and timestamp + HStack(spacing: 6) { + Image(systemName: entry.category.systemImage) + .font(.caption) + .foregroundStyle(.secondary) + Text(entry.description) .font(.caption) .foregroundStyle(.secondary) Spacer() + Button { + store.toggleFavorite(for: entry.id) + } label: { + Image(systemName: entry.isFavorite ? "star.fill" : "star") + .font(.caption) + .foregroundColor(entry.isFavorite ? .yellow : .secondary.opacity(0.4)) + } + .buttonStyle(.plain) + Text(entry.formattedTimestamp) .font(.caption2) .foregroundStyle(.tertiary) diff --git a/TransFrame/Features/MenuBar/MenuBarController.swift b/TransFrame/Features/MenuBar/MenuBarController.swift index 0a1796f..21daac4 100644 --- a/TransFrame/Features/MenuBar/MenuBarController.swift +++ b/TransFrame/Features/MenuBar/MenuBarController.swift @@ -104,6 +104,27 @@ final class MenuBarController { menu.addItem(NSMenuItem.separator()) + // Color Picker + menu.addItem(createMenuItem( + titleKey: "menu.pick.color", + comment: "Pick Color from Screen", + action: #selector(AppDelegate.pickScreenColor), + target: appDelegate, + imageName: "eyedropper" + )) + + // Open Local Image + menu.addItem(createMenuItem( + titleKey: "menu.open.image", + comment: "Open Image for Editing...", + action: #selector(AppDelegate.openImageForEditing), + keyEquivalent: "o", + target: appDelegate, + imageName: "photo.badge.plus" + )) + + menu.addItem(NSMenuItem.separator()) + // Translation History menu.addItem(createMenuItem( titleKey: "menu.translation.history", diff --git a/TransFrame/Features/Pinned/PinnedWindow.swift b/TransFrame/Features/Pinned/PinnedWindow.swift index 6daa296..a5d24d3 100644 --- a/TransFrame/Features/Pinned/PinnedWindow.swift +++ b/TransFrame/Features/Pinned/PinnedWindow.swift @@ -7,6 +7,10 @@ import Combine final class PinnedWindow: NSPanel { // MARK: - Properties + /// Height reserved for the bottom floating toolbar (content spacing + bar). + /// Window frames and default sizes all reserve this so content is never clipped. + private static let toolbarReservedHeight: CGFloat = 46 + let documentID: UUID private(set) var currentDocument: CaptureDocument? private var cancellables = Set() @@ -20,6 +24,9 @@ final class PinnedWindow: NSPanel { /// Current zoom scale factor private(set) var zoomScale: CGFloat = 1.0 + /// Accumulated precise scroll delta waiting to trigger a zoom step + private var pendingZoomDelta: CGFloat = 0 + /// Base initial frame before zooming private var baseFrame: NSRect = .zero @@ -33,14 +40,17 @@ final class PinnedWindow: NSPanel { let primaryHeight = NSScreen.screens.first(where: { $0.frame.origin == .zero })?.frame.height ?? 1080 let cocoaRect = CoordinateMapper.quartzToCocoa(document.sourceScreenRect, primaryHeight: primaryHeight).rawValue - // In-place 1:1 screen frame + // In-place 1:1 screen frame + toolbar space below let initialFrame: NSRect if cocoaRect.width > 10 && cocoaRect.height > 10 { - initialFrame = cocoaRect + var f = cocoaRect + f.size.height += Self.toolbarReservedHeight + f.origin.y -= Self.toolbarReservedHeight + initialFrame = f } else { let defaultSize = NSSize( - width: document.visiblePixelSize.width / 2.0, - height: document.visiblePixelSize.height / 2.0 + width: max(document.visiblePixelSize.width / 2.0, 220), + height: max(document.visiblePixelSize.height / 2.0 + Self.toolbarReservedHeight, 160) ) initialFrame = Self.calculateCenteredRect(size: defaultSize) } @@ -68,7 +78,8 @@ final class PinnedWindow: NSPanel { width: CGFloat(screenshot.image.width) / scaleFactor, height: CGFloat(screenshot.image.height) / scaleFactor ) - let windowSize = Self.calculateWindowSize(for: imageSize) + var windowSize = Self.calculateWindowSize(for: imageSize) + windowSize.height += Self.toolbarReservedHeight let contentRect = Self.calculateCenteredRect(size: windowSize) self.baseFrame = contentRect @@ -102,12 +113,29 @@ final class PinnedWindow: NSPanel { guard let doc = currentDocument else { return } let contentView = PinnedDocumentContent( document: doc, + isPinnedAlwaysOnTop: level == .floating, onClose: { [weak self] in self?.close() }, + onEdit: { [weak self] in + guard let self, let doc = self.currentDocument else { return } + self.close() + PreviewWindowController.shared.showPreview(for: doc.toLegacyScreenshot()) + }, + onToggleAlwaysOnTop: { [weak self] in + guard let self else { return } + if self.level == .floating { + self.level = .normal + } else { + self.level = .floating + } + }, onToggleClickThrough: { [weak self] in self?.toggleClickThrough() }, + onCopy: { [weak self] in + self?.copyRenderedImageToPasteboard() + }, onOpacityChange: { [weak self] alpha in self?.alphaValue = alpha }, @@ -213,6 +241,70 @@ final class PinnedWindow: NSPanel { override var canBecomeKey: Bool { !ignoresMouseEvents } override var canBecomeMain: Bool { false } + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 || event.keyCode == 51 { // Esc or Delete + close() + return + } + // 无修饰键的 E 才触发编辑;⌘E/⌥E 等组合留给系统语义 + let bareModifiers = event.modifierFlags + .intersection(.deviceIndependentFlagsMask) + .subtracting([.capsLock]) + if bareModifiers.isEmpty, event.charactersIgnoringModifiers?.lowercased() == "e" { + if let doc = currentDocument { + close() + PreviewWindowController.shared.showPreview(for: doc.toLegacyScreenshot()) + return + } + } + // ⌘C 复制渲染图(与工具栏复制按钮一致) + if event.modifierFlags.contains(.command), + event.charactersIgnoringModifiers?.lowercased() == "c" { + copyRenderedImageToPasteboard() + return + } + super.keyDown(with: event) + } + + override func scrollWheel(with event: NSEvent) { + if event.modifierFlags.contains(.option) { + // Option + Scroll Wheel: Adjust opacity between 0.2 and 1.0 + let delta = event.scrollingDeltaY * 0.01 + let newAlpha = max(0.2, min(1.0, alphaValue + delta)) + alphaValue = newAlpha + } else if event.hasPreciseScrollingDeltas { + // 触控板精确滚动:累积达到阈值才缩放一步,避免惯性滚动过度灵敏 + pendingZoomDelta += event.scrollingDeltaY + if abs(pendingZoomDelta) >= 10 { + applyZoom(zoomScale * (pendingZoomDelta > 0 ? 1.05 : 0.95)) + pendingZoomDelta = 0 + } + } else { + // 普通滚轮(离散步进) + let zoomDelta: CGFloat = event.scrollingDeltaY > 0 ? 1.05 : 0.95 + applyZoom(zoomScale * zoomDelta) + } + } + + /// Copies the rendered document (with annotations) to the pasteboard, + /// shared by the toolbar button and the ⌘C key equivalent. + @MainActor + private func copyRenderedImageToPasteboard() { + guard let doc = currentDocument, + let cgImg = DocumentRenderer.shared.render(document: doc) else { return } + let nsImg = NSImage(cgImage: cgImg, size: .zero) + NSPasteboard.general.clearContents() + NSPasteboard.general.writeObjects([nsImg]) + } + + override func mouseUp(with event: NSEvent) { + super.mouseUp(with: event) + if event.clickCount == 2 { + // Double click: Reset to 100% original size + applyZoom(1.0) + } + } + override func close() { if !isProgrammaticClose { onClose?() @@ -232,18 +324,26 @@ final class PinnedWindow: NSPanel { private struct PinnedDocumentContent: View { let document: CaptureDocument + var isPinnedAlwaysOnTop: Bool let onClose: () -> Void + let onEdit: () -> Void + let onToggleAlwaysOnTop: () -> Void let onToggleClickThrough: () -> Void + let onCopy: () -> Void let onOpacityChange: (CGFloat) -> Void let onZoomChange: (CGFloat) -> Void - @State private var isHovering = false - @State private var currentOpacity: CGFloat = 1.0 - @State private var currentZoom: CGFloat = 1.0 + @State private var showCopiedToast = false + @State private var alwaysOnTopState: Bool = true @State private var renderedImage: NSImage? var body: some View { - ZStack(alignment: .topTrailing) { + VStack(spacing: 6) { + // 校准置顶状态为窗口真实 level(视图重建/外部变更后保持一致) + EmptyView() + .onAppear { alwaysOnTopState = isPinnedAlwaysOnTop } + + // Pinned Image Group { if let rendered = renderedImage { Image(nsImage: rendered) @@ -253,47 +353,92 @@ private struct PinnedDocumentContent: View { Color.black.opacity(0.1) } } - .clipShape(RoundedRectangle(cornerRadius: 6)) + .clipShape(RoundedRectangle(cornerRadius: 8)) .overlay( - RoundedRectangle(cornerRadius: 6) + RoundedRectangle(cornerRadius: 8) .stroke(Color.white.opacity(0.2), lineWidth: 1) ) - .shadow(color: .black.opacity(0.3), radius: 6, x: 0, y: 3) - - // Hover Controls Toolbar - if isHovering { - HStack(spacing: 8) { - // Click-through button - Button(action: onToggleClickThrough) { - Image(systemName: "cursorarrow.slash") - .font(.system(size: 11, weight: .bold)) - .foregroundColor(.white) + .shadow(color: .black.opacity(0.35), radius: 8, x: 0, y: 3) + + // Bottom Permanent Floating Action Toolbar + HStack(spacing: 8) { + // Edit + Button(action: onEdit) { + HStack(spacing: 4) { + Image(systemName: "pencil.and.outline") + Text("编辑") } - .buttonStyle(.plain) - .help("点击穿透 (快捷键)") - - // Close button - Button(action: onClose) { - Image(systemName: "xmark") - .font(.system(size: 11, weight: .bold)) - .foregroundColor(.white) + .font(.system(size: 11, weight: .medium)) + .foregroundStyle(.primary) + } + .buttonStyle(.plain) + .padding(.horizontal, 6) + .padding(.vertical, 3) + .background(Color.primary.opacity(0.08), in: RoundedRectangle(cornerRadius: 6)) + .customTooltip("在无边框编辑器中编辑 (E)") + + Divider().frame(height: 12) + + // Always on top toggle + Button(action: { + alwaysOnTopState.toggle() + onToggleAlwaysOnTop() + }) { + Image(systemName: alwaysOnTopState ? "pin.fill" : "pin.slash") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(alwaysOnTopState ? .accentColor : .secondary) + } + .buttonStyle(.plain) + .padding(3) + .customTooltip(alwaysOnTopState ? "当前:置顶 (点击取消置顶)" : "当前:普通窗口 (点击置顶)") + + // Click-through toggle + Button(action: onToggleClickThrough) { + Image(systemName: "cursorarrow.slash") + .font(.system(size: 12, weight: .medium)) + .foregroundColor(.secondary) + } + .buttonStyle(.plain) + .padding(3) + .customTooltip("鼠标穿透") + + // Copy to clipboard + Button(action: { + onCopy() + showCopiedToast = true + DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { + showCopiedToast = false } - .buttonStyle(.plain) - .help("关闭钉图") + }) { + Image(systemName: showCopiedToast ? "checkmark" : "doc.on.clipboard") + .font(.system(size: 12, weight: .bold)) + .foregroundColor(showCopiedToast ? .green : .secondary) } - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background(.ultraThinMaterial, in: Capsule()) - .padding(8) - .transition(.opacity.combined(with: .scale(scale: 0.9))) - } - } - .onHover { hovering in - withAnimation(.easeInOut(duration: 0.15)) { - isHovering = hovering + .buttonStyle(.plain) + .padding(3) + .customTooltip("复制图片 (⌘C)") + + Divider().frame(height: 12) + + // Close / Remove + Button(action: onClose) { + Image(systemName: "xmark") + .font(.system(size: 11, weight: .bold)) + .foregroundColor(.red.opacity(0.85)) + } + .buttonStyle(.plain) + .padding(3) + .customTooltip("关闭钉图 (Esc / Delete)") } + .cursor(.arrow) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background(.ultraThinMaterial, in: Capsule()) + .shadow(color: .black.opacity(0.25), radius: 6, y: 2) } + .padding(6) .onAppear { + alwaysOnTopState = isPinnedAlwaysOnTop renderContent() } .onChange(of: document.editRevision) { _, _ in diff --git a/TransFrame/Features/Pinned/PinnedWindowsManager.swift b/TransFrame/Features/Pinned/PinnedWindowsManager.swift index a609976..3ceb1e2 100644 --- a/TransFrame/Features/Pinned/PinnedWindowsManager.swift +++ b/TransFrame/Features/Pinned/PinnedWindowsManager.swift @@ -86,6 +86,30 @@ final class PinnedWindowsManager { return pinnedWindow } + /// Pins an NSImage directly. + /// The backing scale is derived from the NSImage's point size vs its + /// pixel width, so callers that build the image at display scale (e.g. + /// the bilingual result pin button) pin at the correct size on both 1x + /// and 2x screens instead of a hardcoded 2.0. + @discardableResult + func pin(image: NSImage) -> PinnedWindow? { + guard let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) else { return nil } + let pointWidth = max(image.size.width, 1) + let scale = CGFloat(cgImage.width) / pointWidth + let displayScale = max(1, scale.rounded()) + // 屏幕矩形用点尺寸(像素 / backing scale),否则 2x 屏上钉图会显示成两倍大 + let pointSize = CGSize( + width: CGFloat(cgImage.width) / displayScale, + height: CGFloat(cgImage.height) / displayScale + ) + let doc = CaptureDocument( + baseImage: cgImage, + sourceScreenRect: QuartzGlobalRect(CGRect(origin: .zero, size: pointSize)), + sourceDisplay: DisplayInfo(id: 0, name: "Main", frame: CGRect(origin: .zero, size: pointSize), scaleFactor: displayScale, isPrimary: true) + ) + return pin(document: doc) + } + /// Unpins a window by its ID. func unpinWindow(_ id: UUID) { guard let window = pinnedWindows[id] else { return } diff --git a/TransFrame/Features/Preview/AnnotationCanvas.swift b/TransFrame/Features/Preview/AnnotationCanvas.swift index d5ba5e1..4cd9541 100644 --- a/TransFrame/Features/Preview/AnnotationCanvas.swift +++ b/TransFrame/Features/Preview/AnnotationCanvas.swift @@ -54,6 +54,9 @@ struct AnnotationCanvas: View { /// Index of the selected annotation (nil = none selected) var selectedIndex: Int? + /// Index of the annotation currently being edited for text (nil = none) + var editingIndex: Int? + /// The original image for mosaic effect var sourceImage: CGImage? @@ -63,6 +66,19 @@ struct AnnotationCanvas: View { Canvas { context, size in // Draw all completed annotations for (index, annotation) in annotations.enumerated() { + if index == editingIndex { + if case .text = annotation { + // Hide original text while in-place TextField is active + continue + } else if case .calloutBubble(var bubble) = annotation { + bubble.text = "" + drawCalloutBubble(bubble, in: &context, size: size) + if index == selectedIndex { + drawSelectionIndicator(for: annotation, in: &context, size: size) + } + continue + } + } drawAnnotation(annotation, in: &context, size: size) // Draw selection indicator if this annotation is selected @@ -198,14 +214,21 @@ struct AnnotationCanvas: View { let scaledBubbleRect = scaleRect(annotation.rect) let scaledTarget = scalePoint(annotation.targetPoint) - // Draw pointer tail line + // Draw pointer tail triangle + line + let attachX = max(scaledBubbleRect.minX + 10 * scale, min(scaledBubbleRect.maxX - 10 * scale, scaledTarget.x)) + let attachY = scaledTarget.y < scaledBubbleRect.minY ? scaledBubbleRect.minY : (scaledTarget.y > scaledBubbleRect.maxY ? scaledBubbleRect.maxY : scaledBubbleRect.midY) + let halfBase = min(8.0 * scale, max(4.0 * scale, scaledBubbleRect.width * 0.1)) + + var tailTriangle = Path() + tailTriangle.move(to: scaledTarget) + tailTriangle.addLine(to: CGPoint(x: attachX - halfBase, y: attachY)) + tailTriangle.addLine(to: CGPoint(x: attachX + halfBase, y: attachY)) + tailTriangle.closeSubpath() + context.fill(tailTriangle, with: .color(annotation.bubbleColor.color)) + var tailPath = Path() tailPath.move(to: scaledTarget) - let attachPoint = CGPoint( - x: max(scaledBubbleRect.minX + 8 * scale, min(scaledBubbleRect.maxX - 8 * scale, scaledTarget.x)), - y: scaledTarget.y < scaledBubbleRect.minY ? scaledBubbleRect.minY : scaledBubbleRect.maxY - ) - tailPath.addLine(to: attachPoint) + tailPath.addLine(to: CGPoint(x: attachX, y: attachY)) context.stroke( tailPath, with: .color(annotation.bubbleColor.color), @@ -217,15 +240,17 @@ struct AnnotationCanvas: View { context.fill(bubblePath, with: .color(annotation.bubbleColor.color)) // Draw text inside bubble - let font = Font.system(size: max(10, annotation.style.fontSize * scale), weight: .medium) - let text = Text(annotation.text) - .font(font) - .foregroundColor(annotation.style.color.color) - - context.draw( - context.resolve(text), - in: scaledBubbleRect.insetBy(dx: 6 * scale, dy: 6 * scale) - ) + if !annotation.text.isEmpty { + let font = Font.system(size: max(10, annotation.style.fontSize * scale), weight: .medium) + let text = Text(annotation.text) + .font(font) + .foregroundColor(annotation.style.color.color) + + context.draw( + context.resolve(text), + in: scaledBubbleRect.insetBy(dx: 6 * scale, dy: 6 * scale) + ) + } } /// Draws a rectangle annotation @@ -238,10 +263,10 @@ struct AnnotationCanvas: View { let path = Path(scaledRect) if annotation.isFilled { - // Filled rectangle - solid color to hide underlying content + // Filled rectangle - background fill color (falls back to stroke color for legacy docs) context.fill( path, - with: .color(annotation.style.color.color) + with: .color(annotation.effectiveFillColor.color) ) } else { // Hollow rectangle - outline only @@ -291,10 +316,14 @@ struct AnnotationCanvas: View { let lineWidth = annotation.style.lineWidth * scale let color = annotation.style.color.color - // Draw the main line + // Draw the main line (curved if controlPoint is set) var linePath = Path() linePath.move(to: scaledStart) - linePath.addLine(to: scaledEnd) + if let cp = annotation.controlPoint { + linePath.addQuadCurve(to: scaledEnd, control: scalePoint(cp)) + } else { + linePath.addLine(to: scaledEnd) + } context.stroke( linePath, @@ -310,9 +339,10 @@ struct AnnotationCanvas: View { let arrowHeadLength = max(lineWidth * 4, 12 * scale) let arrowHeadAngle: CGFloat = .pi / 6 // 30 degrees - // Calculate the angle of the line - let dx = scaledEnd.x - scaledStart.x - let dy = scaledEnd.y - scaledStart.y + // Calculate tangent angle at the arrowhead + let prevPoint = annotation.controlPoint.map { scalePoint($0) } ?? scaledStart + let dx = scaledEnd.x - prevPoint.x + let dy = scaledEnd.y - prevPoint.y let angle = atan2(dy, dx) // Calculate arrowhead points @@ -343,19 +373,17 @@ struct AnnotationCanvas: View { ) { guard !annotation.content.isEmpty else { return } - let scaledPoint = scalePoint(annotation.position) - let scaledFontSize = annotation.style.fontSize * scale - + // Draw inside the measured bounds so wrapped lines are fully visible + let scaledBounds = scaleRect(annotation.bounds) let text = Text(annotation.content) .font(annotation.style.fontName == ".AppleSystemUIFont" - ? .system(size: scaledFontSize) - : .custom(annotation.style.fontName, size: scaledFontSize)) + ? .system(size: annotation.style.fontSize * scale) + : .custom(annotation.style.fontName, size: annotation.style.fontSize * scale)) .foregroundColor(annotation.style.color.color) context.draw( context.resolve(text), - at: scaledPoint, - anchor: .topLeading + in: scaledBounds ) } @@ -369,10 +397,10 @@ struct AnnotationCanvas: View { let path = Path(ellipseIn: scaledRect) if annotation.isFilled { - // Filled ellipse - solid color to hide underlying content + // Filled ellipse - background fill color (falls back to stroke color for legacy docs) context.fill( path, - with: .color(annotation.style.color.color) + with: .color(annotation.effectiveFillColor.color) ) } else { // Hollow ellipse - outline only @@ -532,6 +560,21 @@ struct AnnotationCanvas: View { let scaledSize = annotation.size * scale let scaledRadius = scaledSize / 2 + // If there is a lead point, draw the connecting line and pointer dot + if let lp = annotation.leadPoint { + let scaledLead = scalePoint(lp) + var leadPath = Path() + leadPath.move(to: scaledPoint) + leadPath.addLine(to: scaledLead) + context.stroke( + leadPath, + with: .color(annotation.color.color), + style: SwiftUI.StrokeStyle(lineWidth: 2 * scale, lineCap: .round) + ) + let dotRect = CGRect(x: scaledLead.x - 3 * scale, y: scaledLead.y - 3 * scale, width: 6 * scale, height: 6 * scale) + context.fill(Path(ellipseIn: dotRect), with: .color(annotation.color.color)) + } + // Draw circle background let circleRect = CGRect( x: scaledPoint.x - scaledRadius, @@ -560,35 +603,33 @@ struct AnnotationCanvas: View { in context: inout GraphicsContext, size: CGSize ) { + let paddingInImageCoords: CGFloat = 4 let bounds = annotation.bounds - let scaledBounds = scaleRect(bounds) - - // Add padding around the bounds - let padding: CGFloat = 4 * scale - let selectionRect = scaledBounds.insetBy(dx: -padding, dy: -padding) + let scaledBounds = scaleRect(bounds.insetBy(dx: -paddingInImageCoords, dy: -paddingInImageCoords)) - // Draw selection border (dashed blue line) - let borderPath = Path(roundedRect: selectionRect, cornerRadius: 3 * scale) - context.stroke( - borderPath, - with: .color(.accentColor), - style: SwiftUI.StrokeStyle( - lineWidth: 2, - lineCap: .round, - dash: [6, 4] + // Draw selection border (dashed blue line) for box-like annotations + switch annotation { + case .line, .arrow: + break + default: + let borderPath = Path(roundedRect: scaledBounds, cornerRadius: 3 * scale) + context.stroke( + borderPath, + with: .color(.accentColor), + style: SwiftUI.StrokeStyle( + lineWidth: 1.5, + lineCap: .round, + dash: [6, 4] + ) ) - ) + } - // Draw corner handles - let handleSize: CGFloat = 8 - let handlePositions = [ - CGPoint(x: selectionRect.minX, y: selectionRect.minY), // Top-left - CGPoint(x: selectionRect.maxX, y: selectionRect.minY), // Top-right - CGPoint(x: selectionRect.minX, y: selectionRect.maxY), // Bottom-left - CGPoint(x: selectionRect.maxX, y: selectionRect.maxY) // Bottom-right - ] + // Draw interactive control handles + let handleSize: CGFloat = 9 + let handles = annotation.controlHandles(padding: paddingInImageCoords) - for position in handlePositions { + for (handle, imagePoint) in handles { + let position = scalePoint(imagePoint) let handleRect = CGRect( x: position.x - handleSize / 2, y: position.y - handleSize / 2, @@ -597,9 +638,15 @@ struct AnnotationCanvas: View { ) let handlePath = Path(ellipseIn: handleRect) - // White fill with blue border - context.fill(handlePath, with: .color(.white)) - context.stroke(handlePath, with: .color(.accentColor), lineWidth: 2) + if handle.isSpecialPoint { + // Special handles (curve controlPoint, leadPoint, targetPoint) rendered in yellow/orange with accent border + context.fill(handlePath, with: .color(.yellow)) + context.stroke(handlePath, with: .color(.accentColor), lineWidth: 2) + } else { + // Standard resize/endpoint handles: white fill with accent border + context.fill(handlePath, with: .color(.white)) + context.stroke(handlePath, with: .color(.accentColor), lineWidth: 2) + } } } diff --git a/TransFrame/Features/Preview/PreviewActionButtons.swift b/TransFrame/Features/Preview/PreviewActionButtons.swift index 66ff6b9..56aa865 100644 --- a/TransFrame/Features/Preview/PreviewActionButtons.swift +++ b/TransFrame/Features/Preview/PreviewActionButtons.swift @@ -32,6 +32,18 @@ struct PreviewActionButtons: View { ocrButton + Divider() + .frame(height: 16) + .accessibilityHidden(true) + + translateButton + + Divider() + .frame(height: 16) + .accessibilityHidden(true) + + smartMosaicButton + Divider() .frame(height: 16) .accessibilityHidden(true) @@ -56,7 +68,7 @@ struct PreviewActionButtons: View { : Color.clear ) .clipShape(RoundedRectangle(cornerRadius: 4)) - .help(String(localized: "preview.tooltip.crop")) + .customTooltip(String(localized: "preview.tooltip.crop")) .accessibilityLabel(Text("preview.crop")) .accessibilityHint(Text("Press C to toggle")) } @@ -74,7 +86,7 @@ struct PreviewActionButtons: View { : Color.clear ) .clipShape(RoundedRectangle(cornerRadius: 4)) - .help(String(localized: "preview.tooltip.pin")) + .customTooltip(String(localized: "preview.tooltip.pin")) .accessibilityLabel(Text("preview.pin")) .accessibilityHint(Text("Press P to pin")) } @@ -87,7 +99,7 @@ struct PreviewActionButtons: View { Image(systemName: "arrow.uturn.backward") } .disabled(!viewModel.canUndo) - .help(String(localized: "preview.tooltip.undo")) + .customTooltip(String(localized: "preview.tooltip.undo")) .accessibilityLabel(Text("action.undo")) .accessibilityHint(Text("Command Z")) @@ -97,7 +109,7 @@ struct PreviewActionButtons: View { Image(systemName: "arrow.uturn.forward") } .disabled(!viewModel.canRedo) - .help(String(localized: "preview.tooltip.redo")) + .customTooltip(String(localized: "preview.tooltip.redo")) .accessibilityLabel(Text("action.redo")) .accessibilityHint(Text("Command Shift Z")) } @@ -114,7 +126,7 @@ struct PreviewActionButtons: View { } } .disabled(viewModel.isSaving) - .help(String(localized: "preview.tooltip.save")) + .customTooltip(String(localized: "preview.tooltip.save")) .accessibilityLabel(Text(String(localized: viewModel.isSaving ? "preview.accessibility.saving" : "preview.accessibility.save"))) .accessibilityHint(Text(String(localized: "preview.accessibility.hint.commandS"))) } @@ -132,7 +144,41 @@ struct PreviewActionButtons: View { } } .disabled(viewModel.isPerformingOCR) - .help(String(localized: "preview.tooltip.ocr")) + .customTooltip(String(localized: "preview.tooltip.ocr")) + } + + /// Translation: runs the shared pipeline and shows results in the embedded drawer + private var translateButton: some View { + Button { + viewModel.performTranslation() + } label: { + if viewModel.isTranslating { + ProgressView() + .controlSize(.small) + .frame(width: 16, height: 16) + } else { + Image(systemName: "character.book.closed") + } + } + .disabled(viewModel.isTranslating) + .customTooltip(String(localized: "preview.panel.translate")) + } + + /// Smart mosaic: detects sensitive privacy regions and masks them in one batch + private var smartMosaicButton: some View { + Button { + viewModel.performSmartRedaction() + } label: { + if viewModel.isDetectingPrivacy { + ProgressView() + .controlSize(.small) + .frame(width: 16, height: 16) + } else { + Image(systemName: "camera.metering.unknown") + } + } + .disabled(viewModel.isDetectingPrivacy) + .customTooltip(String(localized: "preview.tooltip.smartMosaic")) } /// Confirm button: copies to clipboard and dismisses (only on success) @@ -149,7 +195,7 @@ struct PreviewActionButtons: View { } } .disabled(viewModel.isCopying) - .help(String(localized: "preview.tooltip.confirm")) + .customTooltip(String(localized: "preview.tooltip.confirm")) .accessibilityLabel(Text(String(localized: viewModel.isCopying ? "preview.accessibility.copying" : "preview.accessibility.confirm"))) .accessibilityHint(Text(String(localized: "preview.accessibility.hint.enter"))) } diff --git a/TransFrame/Features/Preview/PreviewAnnotatedImageView.swift b/TransFrame/Features/Preview/PreviewAnnotatedImageView.swift index 2a53538..30b8460 100644 --- a/TransFrame/Features/Preview/PreviewAnnotatedImageView.swift +++ b/TransFrame/Features/Preview/PreviewAnnotatedImageView.swift @@ -7,6 +7,14 @@ struct PreviewAnnotatedImageView: View { @Binding var imageScale: CGFloat @FocusState.Binding var isTextFieldFocused: Bool + @State private var hoverImageLocation: CGPoint? + @State private var didBeginGesture: Bool = false + /// Set when the click that committed text input must not also draw: + /// the rest of that gesture is swallowed (OCR review). + @State private var didInvalidateGesture: Bool = false + @State private var lastTapTime: Date = .distantPast + @State private var lastTappedAnnotationIndex: Int? + private var displayScaleFactor: CGFloat { viewModel.screenshot.sourceDisplay.scaleFactor } @@ -25,21 +33,23 @@ struct PreviewAnnotatedImageView: View { "Screenshot preview, \(viewModel.dimensionsText), from \(viewModel.displayName)" )) + if !viewModel.isTranslating && viewModel.hasTranslationResults { + BilingualOverlayView(segments: viewModel.bilingualSegments) + .frame(width: imageSize.width, height: imageSize.height) + .clipped() + } + AnnotationCanvas( annotations: viewModel.annotations, currentAnnotation: viewModel.currentAnnotation, canvasSize: imageSize, scale: 1.0 / displayScaleFactor, selectedIndex: viewModel.selectedAnnotationIndex, + editingIndex: viewModel.isWaitingForTextInput ? viewModel.editingAnnotationIndex : nil, sourceImage: viewModel.image ) .frame(width: imageSize.width, height: imageSize.height) - if viewModel.isWaitingForTextInput, - let inputPosition = viewModel.textInputPosition { - textInputField(at: inputPosition) - } - if viewModel.selectedTool != nil { drawingGestureOverlay } @@ -48,6 +58,12 @@ struct PreviewAnnotatedImageView: View { selectionGestureOverlay } + if viewModel.isWaitingForTextInput, + let inputPosition = viewModel.textInputPosition { + textInputField(at: inputPosition) + .zIndex(10) + } + if viewModel.isCropMode { PreviewCropOverlay(viewModel: viewModel, displaySize: imageSize, scale: 1.0 / displayScaleFactor) } @@ -56,9 +72,11 @@ struct PreviewAnnotatedImageView: View { if let tool = viewModel.selectedTool { activeToolIndicator(tool: tool) .padding(8) + .allowsHitTesting(false) } else if viewModel.isCropMode { cropModeIndicator .padding(8) + .allowsHitTesting(false) } } .overlay(alignment: .bottom) { @@ -72,6 +90,14 @@ struct PreviewAnnotatedImageView: View { imageDisplaySize = imageSize imageScale = 1.0 } + .onContinuousHover { phase in + switch phase { + case .active(let location): + hoverImageLocation = convertToImageCoordinates(location) + case .ended: + hoverImageLocation = nil + } + } .contentShape(Rectangle()) .cursor(cursorForCurrentTool) } @@ -81,21 +107,51 @@ struct PreviewAnnotatedImageView: View { return .crosshair } - guard let tool = viewModel.selectedTool else { - if viewModel.isDraggingAnnotation { - return .closedHand - } else if viewModel.selectedAnnotationIndex != nil { - return .openHand + if let activeHandle = viewModel.activeResizeHandle { + return activeHandle.cursor + } + + if viewModel.isDraggingAnnotation { + return .closedHand + } + + if let hoverPt = hoverImageLocation { + if let handle = viewModel.hitTestSelectedAnnotationHandle(at: hoverPt) { + return handle.cursor + } + + if let tool = viewModel.selectedTool { + if (tool == .text || tool == .calloutBubble), + let hitIdx = viewModel.hitTest(at: hoverPt) { + switch viewModel.annotations[hitIdx] { + case .text, .calloutBubble: + return .iBeam + default: + break + } + } + if let selIdx = viewModel.selectedAnnotationIndex, + viewModel.hitTest(at: hoverPt) == selIdx { + return .openHand + } + return tool.cursor + } else { + if let hitIdx = viewModel.hitTest(at: hoverPt) { + if case .text = viewModel.annotations[hitIdx], + viewModel.selectedAnnotationIndex == hitIdx { + return .openHand + } + return .openHand + } + return .arrow } - return .arrow } - switch tool { - case .rectangle, .ellipse, .line, .freehand, .arrow, .highlight, .mosaic, .numberLabel, .privacyRedaction, .calloutBubble: - return .crosshair - case .text: - return .iBeam + if let tool = viewModel.selectedTool { + return tool.cursor } + + return .arrow } private var drawingGestureOverlay: some View { @@ -104,16 +160,72 @@ struct PreviewAnnotatedImageView: View { .gesture( DragGesture(minimumDistance: 0) .onChanged { value in - let point = convertToImageCoordinates(value.location) - if value.translation == .zero { - viewModel.beginDrawing(at: point) + if didInvalidateGesture { return } + let startPoint = convertToImageCoordinates(value.startLocation) + let currentPoint = convertToImageCoordinates(value.location) + + if !didBeginGesture { + didBeginGesture = true + // Commit any active text input if user clicks elsewhere on canvas + if viewModel.isWaitingForTextInput { + viewModel.commitTextInput() + isTextFieldFocused = false + // If the user was just finishing text input by clicking canvas, don't immediately spawn a new box at that click unless they drag or click again + if viewModel.selectedTool == .text { + // Swallow the rest of this gesture so the + // commit click cannot start a stray drawing + // mid-gesture (didBeginGesture is already true). + didInvalidateGesture = true + return + } + } + if let handle = viewModel.hitTestSelectedAnnotationHandle(at: startPoint) { + viewModel.beginResizingAnnotation(handle: handle, at: startPoint) + return + } + // Clicking an existing annotation selects and drags it; + // double click re-edits text/callout content in place + if let hitIdx = viewModel.hitTest(at: startPoint) { + let now = Date() + let isDoubleTap = now.timeIntervalSince(lastTapTime) < 0.35 + && lastTappedAnnotationIndex == hitIdx + lastTapTime = now + lastTappedAnnotationIndex = hitIdx + if isDoubleTap { + switch viewModel.annotations[hitIdx] { + case .text, .calloutBubble: + viewModel.beginEditingTextAnnotation(at: hitIdx) + return + default: + break + } + } + viewModel.selectedAnnotationIndex = hitIdx + viewModel.beginDraggingAnnotation(at: startPoint) + return + } + lastTappedAnnotationIndex = nil + viewModel.beginDrawing(at: startPoint) } else { - viewModel.continueDrawing(to: point) + if viewModel.isResizingAnnotation { + viewModel.continueResizingAnnotation(to: currentPoint) + } else if !viewModel.isWaitingForTextInput { + viewModel.continueDrawing(to: currentPoint) + } } } .onEnded { value in - let point = convertToImageCoordinates(value.location) - viewModel.endDrawing(at: point) + defer { + didBeginGesture = false + didInvalidateGesture = false + } + if didInvalidateGesture { return } + let endPoint = convertToImageCoordinates(value.location) + if viewModel.isResizingAnnotation { + viewModel.endResizingAnnotation() + } else if !viewModel.isWaitingForTextInput { + viewModel.endDrawing(at: endPoint) + } } ) } @@ -124,20 +236,59 @@ struct PreviewAnnotatedImageView: View { .gesture( DragGesture(minimumDistance: 0) .onChanged { value in - let point = convertToImageCoordinates(value.location) - if value.translation == .zero { - if let hitIndex = viewModel.hitTest(at: point) { + let startPoint = convertToImageCoordinates(value.startLocation) + let currentPoint = convertToImageCoordinates(value.location) + + if !didBeginGesture { + didBeginGesture = true + + if viewModel.isWaitingForTextInput { + viewModel.commitTextInput() + isTextFieldFocused = false + } + + // 1. Check if clicking a resize/control handle on currently selected annotation + if let handle = viewModel.hitTestSelectedAnnotationHandle(at: startPoint) { + viewModel.beginResizingAnnotation(handle: handle, at: startPoint) + return + } + + // 2. Check if clicking an existing annotation + if let hitIndex = viewModel.hitTest(at: startPoint) { + let now = Date() + if now.timeIntervalSince(lastTapTime) < 0.35 && lastTappedAnnotationIndex == hitIndex { + switch viewModel.annotations[hitIndex] { + case .text, .calloutBubble: + lastTappedAnnotationIndex = nil + viewModel.beginEditingTextAnnotation(at: hitIndex) + return + default: + break + } + } + lastTapTime = now + lastTappedAnnotationIndex = hitIndex viewModel.selectAnnotation(at: hitIndex) - viewModel.beginDraggingAnnotation(at: point) + viewModel.beginDraggingAnnotation(at: startPoint) } else { + lastTappedAnnotationIndex = nil viewModel.deselectAnnotation() } - } else if viewModel.isDraggingAnnotation { - viewModel.continueDraggingAnnotation(to: point) + } else { + if viewModel.isResizingAnnotation { + viewModel.continueResizingAnnotation(to: currentPoint) + } else if viewModel.isDraggingAnnotation { + viewModel.continueDraggingAnnotation(to: currentPoint) + } } } .onEnded { _ in - viewModel.endDraggingAnnotation() + defer { didBeginGesture = false } + if viewModel.isResizingAnnotation { + viewModel.endResizingAnnotation() + } else if viewModel.isDraggingAnnotation { + viewModel.endDraggingAnnotation() + } } ) } @@ -151,22 +302,67 @@ struct PreviewAnnotatedImageView: View { x: position.x / displayScaleFactor, y: position.y / displayScaleFactor ) - return TextField(String(localized: "preview.enter.text"), text: $viewModel.textInputContent) - .textFieldStyle(.plain) - .font(.system(size: 14)) - .foregroundColor(AppSettings.shared.strokeColor.color) - .padding(4) - .background(Color.white.opacity(0.9)) - .cornerRadius(4) - .frame(minWidth: 100, maxWidth: 300) - .position(x: displayPosition.x + 50, y: displayPosition.y + 10) + + let (fontSizeInImageCoords, textColor): (CGFloat, Color) = { + if let editIdx = viewModel.editingAnnotationIndex, + editIdx >= 0, + editIdx < viewModel.annotations.count { + switch viewModel.annotations[editIdx] { + case .text(let t): + return (t.style.fontSize, t.style.color.color) + case .calloutBubble(let cb): + return (cb.style.fontSize, cb.style.color.color) + default: + break + } + } + return (AppSettings.shared.textSize, AppSettings.shared.strokeColor.color) + }() + + let displayFontSize = max(10, fontSizeInImageCoords / displayScaleFactor) + let clampedX = max(0, min(imageSize.width - 60, displayPosition.x - 4)) + let clampedY = max(0, min(imageSize.height - displayFontSize - 6, displayPosition.y - 2)) + + // Size the editor from the current content so multi-line input is fully visible; + // cap by the space remaining after the clamped origin so the editor never + // overflows the canvas when typing near the right/bottom edge. + let content = viewModel.textInputContent + let lines = content.split(separator: "\n", omittingEmptySubsequences: false) + let lineCount = max(1, lines.count) + let measureFont = NSFont.systemFont(ofSize: displayFontSize) + let longestLineWidth = lines.map { ($0 as NSString).size(withAttributes: [.font: measureFont]).width }.max() ?? 0 + let editorWidth = min(max(longestLineWidth + 24, 60), max(60, imageSize.width - clampedX)) + let editorHeight = min( + CGFloat(lineCount) * max(displayFontSize * 1.45, 20) + 10, + max(30, imageSize.height - clampedY) + ) + + return TextEditor(text: $viewModel.textInputContent) + .font(.system(size: displayFontSize)) + .foregroundColor(textColor) + .scrollContentBackground(.hidden) + .padding(.horizontal, 4) + .padding(.vertical, 2) + .background( + RoundedRectangle(cornerRadius: 4) + .fill(Color(nsColor: .textBackgroundColor).opacity(0.85)) + .overlay( + RoundedRectangle(cornerRadius: 4) + .stroke(Color.accentColor, lineWidth: 1.5) + ) + ) + .frame(width: editorWidth, height: editorHeight, alignment: .topLeading) + .offset(x: clampedX, y: clampedY) .focused($isTextFieldFocused) .onAppear { - isTextFieldFocused = true + DispatchQueue.main.async { + isTextFieldFocused = true + } } - .onSubmit { - viewModel.commitTextInput() - isTextFieldFocused = false + .onChange(of: isTextFieldFocused) { _, focused in + if !focused && viewModel.isWaitingForTextInput { + viewModel.commitTextInput() + } } .onExitCommand { viewModel.cancelCurrentDrawing() diff --git a/TransFrame/Features/Preview/PreviewContentView.swift b/TransFrame/Features/Preview/PreviewContentView.swift index dbc812a..e84be26 100644 --- a/TransFrame/Features/Preview/PreviewContentView.swift +++ b/TransFrame/Features/Preview/PreviewContentView.swift @@ -9,50 +9,123 @@ struct PreviewContentView: View { @FocusState private var isTextFieldFocused: Bool @Environment(\.accessibilityReduceMotion) private var reduceMotion - var body: some View { - VStack(spacing: 0) { - ScrollView([.horizontal, .vertical], showsIndicators: true) { - PreviewAnnotatedImageView( - viewModel: viewModel, - imageDisplaySize: $imageDisplaySize, - imageScale: $imageScale, - isTextFieldFocused: $isTextFieldFocused - ) - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color(nsColor: .windowBackgroundColor)) + /// Whether the right-side results drawer should occupy layout space + private var showsResultsPanel: Bool { + viewModel.hasOCRResults || viewModel.isPerformingOCR + || viewModel.hasTranslationResults || viewModel.isTranslating + || isResultsPanelExpanded + } - Divider() + /// Canvas area + results drawer + floating toolbar (see body note). + private var mainContent: some View { + ZStack(alignment: .bottom) { + // Main Canvas Area + Embedded Right Results Drawer + HStack(spacing: 0) { + ScrollView([.horizontal, .vertical], showsIndicators: false) { + PreviewAnnotatedImageView( + viewModel: viewModel, + imageDisplaySize: $imageDisplaySize, + imageScale: $imageScale, + isTextFieldFocused: $isTextFieldFocused + ) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .padding(16) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) - infoBar - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(.bar) - - if viewModel.hasOCRResults { - Divider() - PreviewResultsPanel(viewModel: viewModel, isExpanded: $isResultsPanelExpanded) - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(.bar) + if showsResultsPanel && isResultsPanelExpanded { + PreviewResultsPanel(viewModel: viewModel, isExpanded: $isResultsPanelExpanded) + .frame(width: 300) + .transition(.move(edge: .trailing).combined(with: .opacity)) + } else if showsResultsPanel { + // Once collapsed, the drawer has no other way back: + // keep a slim handle so results stay reachable. + Button { + isResultsPanelExpanded = true + } label: { + Image(systemName: "sidebar.trailing") + .font(.system(size: 14, weight: .medium)) + .foregroundStyle(.secondary) + .frame(width: 26) + .frame(maxHeight: .infinity) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.panel.expandDrawer"), placement: .top) + .padding(.vertical, 8) + } } - } - .onChange(of: viewModel.hasOCRResults) { _, hasResults in - if hasResults { - isResultsPanelExpanded = true + .animation(reduceMotion ? nil : .easeInOut(duration: 0.25), value: isResultsPanelExpanded) + + // Attached Floating Pill Toolbar + HStack(spacing: 12) { + PreviewToolBar(viewModel: viewModel) + Divider().frame(height: 16) + PreviewActionButtons(viewModel: viewModel) } + .cursor(.arrow) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 10)) + .shadow(color: .black.opacity(0.25), radius: 8, y: 2) + .padding(.bottom, 12) } - .alert( - String(localized: "error.title"), - isPresented: .constant(viewModel.errorMessage != nil), - presenting: viewModel.errorMessage - ) { _ in - Button(String(localized: "button.ok")) { - viewModel.errorMessage = nil - } - } message: { message in - Text(message) + } + + /// Top-right dismiss button (see body note). + private var dismissButton: some View { + Button { + viewModel.dismiss() + } label: { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 20)) + .foregroundStyle(.secondary, Color.black.opacity(0.1)) + .padding(10) } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.tooltip.dismiss"), placement: .bottom) + } + + var body: some View { + // Body is split into sub-properties: the full inline chain exceeded + // the type checker's expression budget (rdar-style "unable to + // type-check in reasonable time"). + mainContent + .overlay(alignment: .topTrailing) { + dismissButton + } + .background( + RoundedRectangle(cornerRadius: 12) + .fill(.ultraThinMaterial) + .shadow(color: .black.opacity(0.35), radius: 16, y: 6) + ) + .onChange(of: viewModel.hasOCRResults) { _, hasResults in + if hasResults { + isResultsPanelExpanded = true + } + } + .onChange(of: viewModel.isTranslating) { _, translating in + if translating { + isResultsPanelExpanded = true + } + } + .overlay(alignment: .top) { + if let message = viewModel.smartRedactionMessage { + SmartRedactionToast(message: message, isDetecting: viewModel.isDetectingPrivacy) + } + } + .animation(.easeInOut(duration: 0.3), value: viewModel.smartRedactionMessage != nil) + .alert( + String(localized: "error.title"), + isPresented: .constant(viewModel.errorMessage != nil), + presenting: viewModel.errorMessage + ) { _ in + Button(String(localized: "button.ok")) { + viewModel.errorMessage = nil + } + } message: { message in + Text(message) + } .alert( String(localized: "save.success.title"), isPresented: .constant(viewModel.saveSuccessMessage != nil), @@ -85,42 +158,30 @@ struct PreviewContentView: View { } .animation(.easeInOut(duration: 0.3), value: viewModel.copySuccessMessage != nil) } +} - private var infoBar: some View { - HStack(spacing: 12) { - HStack(spacing: 8) { - Text(viewModel.dimensionsText) - .font(.system(.caption, design: .monospaced)) - .foregroundStyle(.secondary) - .help(String(localized: "preview.image.dimensions")) - - Text("•") - .foregroundStyle(.tertiary) - - Text(viewModel.fileSizeText) - .font(.system(.caption, design: .monospaced)) - .foregroundStyle(.secondary) - .help(String(localized: "preview.estimated.size")) - } - .fixedSize() - - Divider() - .frame(height: 16) +/// Smart-redaction status toast. Split out of the main body because the +/// inline ternary + modifier chain exceeded the type checker's budget. +private struct SmartRedactionToast: View { + let message: String + let isDetecting: Bool - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 8) { - PreviewToolBar(viewModel: viewModel) - } - .padding(.horizontal, 4) + var body: some View { + HStack(spacing: 8) { + Image(systemName: isDetecting ? "camera.metering.unknown" : "checkmark.circle.fill") + .foregroundColor(isDetecting ? Color.secondary : Color.green) + if isDetecting { + ProgressView().controlSize(.mini) } - .frame(minWidth: 100) - - Divider() - .frame(height: 16) - - PreviewActionButtons(viewModel: viewModel) - .fixedSize() + Text(message) } + .padding(.horizontal, 16) + .padding(.vertical, 10) + .background(.ultraThinMaterial) + .cornerRadius(8) + .shadow(radius: 4) + .padding(.top, 20) + .transition(.move(edge: .top).combined(with: .opacity)) } } diff --git a/TransFrame/Features/Preview/PreviewResultsPanel.swift b/TransFrame/Features/Preview/PreviewResultsPanel.swift index f3283f4..a4e11c0 100644 --- a/TransFrame/Features/Preview/PreviewResultsPanel.swift +++ b/TransFrame/Features/Preview/PreviewResultsPanel.swift @@ -1,56 +1,263 @@ import SwiftUI import AppKit +/// Embedded right-side results drawer shown inside the preview window. +/// Hosts the OCR text tab and the bilingual translation tab. struct PreviewResultsPanel: View { @Bindable var viewModel: PreviewViewModel @Binding var isExpanded: Bool + @State private var copiedFlash: String? + @State private var flashGeneration = 0 + /// Mirror of the target-language setting: AppSettings is not observable, + /// so without this the picker label/checkmark would not refresh when the + /// menu is used while no results trigger a re-render. + @State private var selectedTargetLanguage: TranslationLanguage? + + /// Fixed widths so skeleton rows don't jitter on every re-render. + private static let skeletonWidths: [CGFloat] = [240, 180, 210, 150, 220] + + /// Common target languages for the quick switcher + private static let targetLanguages: [TranslationLanguage] = [ + .chineseSimplified, .english, .japanese, .korean, + .french, .german, .spanish, .russian + ] var body: some View { - VStack(alignment: .leading, spacing: 0) { - Button { - withAnimation(.easeInOut(duration: 0.2)) { - isExpanded.toggle() + VStack(spacing: 0) { + header + + Divider() + + Group { + switch viewModel.resultsPanelTab { + case .ocr: + ocrContent + case .translate: + translateContent } - } label: { - HStack { - Image(systemName: isExpanded ? "chevron.down" : "chevron.right") - .frame(width: 12) - Text(String(localized: "preview.results.panel")) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .top) + .background(.regularMaterial) + } + + // MARK: - Header + + private var header: some View { + HStack(spacing: 8) { + if let flash = copiedFlash { + Text(flash) + .font(.caption2) + .foregroundStyle(.green) + .lineLimit(1) + .transition(.opacity) + } + Spacer(minLength: 0) + // 无 tab 切换:内容随工具栏最后一次触发的动作而定。 + + rerunButton + copyAllButton + closeButton + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + } + + /// Re-runs the action for the active tab + private var rerunButton: some View { + Button { + switch viewModel.resultsPanelTab { + case .ocr: + viewModel.performOCR() + case .translate: + viewModel.performTranslation() + } + } label: { + Image(systemName: "arrow.clockwise") + .font(.system(size: 11)) + .frame(width: 22, height: 22) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.panel.rerun")) + } + + private var copyAllButton: some View { + Button { + copyToClipboard(currentTabText) + } label: { + Image(systemName: "doc.on.doc") + .font(.system(size: 11)) + .frame(width: 22, height: 22) + } + .buttonStyle(.plain) + .disabled(currentTabText.isEmpty) + .customTooltip(String(localized: "preview.panel.copyAll")) + } + + private var closeButton: some View { + Button { + withAnimation(.easeInOut(duration: 0.2)) { + isExpanded = false + } + } label: { + Image(systemName: "xmark") + .font(.system(size: 11)) + .frame(width: 22, height: 22) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.panel.close")) + } + + // MARK: - OCR Tab + + private var ocrContent: some View { + Group { + if viewModel.isPerformingOCR { + VStack(alignment: .leading, spacing: 8) { + ForEach(Self.skeletonWidths, id: \.self) { width in + RoundedRectangle(cornerRadius: 3) + .fill(Color.primary.opacity(0.08)) + .frame(width: width, height: 12) + } + Text(String(localized: "preview.panel.recognizing")) .font(.caption) - .fontWeight(.medium) - Spacer() + .foregroundStyle(.secondary) } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(12) + } else if let result = viewModel.ocrResult, !result.observations.isEmpty { + SelectableTextBlock( + attributedText: ResultsTextBuilder.ocrText( + lines: sortedObservations(result).map(\.text) + ), + onLineCopied: flashCopy + ) + .padding(12) + } else { + emptyState(text: String(localized: "preview.panel.noResults")) + } + } + } + + private func sortedObservations(_ result: OCRResult) -> [OCRText] { + result.observations.sorted { $0.boundingBox.minY < $1.boundingBox.minY } + } + + // MARK: - Translation Tab + + private var translateContent: some View { + Group { + if viewModel.isTranslating { + VStack(alignment: .leading, spacing: 8) { + ForEach(Self.skeletonWidths.dropLast(), id: \.self) { width in + RoundedRectangle(cornerRadius: 3) + .fill(Color.primary.opacity(0.08)) + .frame(width: width, height: 12) + } + Text(String(localized: "preview.panel.translating")) + .font(.caption) + .foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(12) + } else if viewModel.hasTranslationResults { + VStack(alignment: .leading, spacing: 8) { + targetLanguagePicker + SelectableTextBlock( + attributedText: ResultsTextBuilder.translationText(segments: viewModel.bilingualSegments), + onLineCopied: flashCopy + ) + } + .padding(12) + } else { + VStack(spacing: 12) { + emptyState(text: String(localized: "preview.panel.noResults")) + targetLanguagePicker + Button { + viewModel.performTranslation() + } label: { + Label(String(localized: "preview.panel.translate"), systemImage: "character.book.closed") + } + .buttonStyle(.borderedProminent) + .controlSize(.small) + } + .padding(12) + } + } + } + + /// Quick target-language switcher; changing it re-runs the translation + private var targetLanguagePicker: some View { + HStack(spacing: 6) { + Text(String(localized: "preview.panel.targetLanguage")) + .font(.caption) .foregroundStyle(.secondary) - .contentShape(Rectangle()) - } - .buttonStyle(.plain) - - if isExpanded { - VStack(alignment: .leading, spacing: 12) { - if viewModel.hasOCRResults { - VStack(alignment: .leading, spacing: 4) { - HStack { - Text(String(localized: "preview.recognized.text")) - .font(.caption) - .foregroundStyle(.secondary) - Spacer() - Button { - NSPasteboard.general.clearContents() - NSPasteboard.general.setString(viewModel.combinedOCRText, forType: .string) - } label: { - Image(systemName: "doc.on.doc") - .font(.caption) - } - .buttonStyle(.plain) - .help(String(localized: "preview.copy.text")) - } - Text(viewModel.combinedOCRText) - .font(.body) - .textSelection(.enabled) + Menu { + ForEach(Self.targetLanguages, id: \.self) { language in + Button { + guard selectedTargetLanguage != language else { return } + selectedTargetLanguage = language + AppSettings.shared.translationTargetLanguage = language + if viewModel.hasTranslationResults || viewModel.ocrResult != nil { + viewModel.performTranslation() + } + } label: { + if selectedTargetLanguage == language { + Label(language.localizedName, systemImage: "checkmark") + } else { + Text(language.localizedName) } } } - .padding(.top, 8) + } label: { + Text(selectedTargetLanguage?.localizedName ?? "—") + .font(.caption) + } + .controlSize(.small) + .fixedSize() + .onAppear { + if selectedTargetLanguage == nil { + selectedTargetLanguage = AppSettings.shared.translationTargetLanguage + } + } + } + } + + // MARK: - Helpers + + private var currentTabText: String { + switch viewModel.resultsPanelTab { + case .ocr: + return viewModel.combinedOCRText + case .translate: + return viewModel.bilingualSegments.map(\.translated).joined(separator: "\n") + } + } + + private func emptyState(text: String) -> some View { + Text(text) + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity) + .multilineTextAlignment(.center) + .padding(24) + } + + private func copyToClipboard(_ text: String) { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(text, forType: .string) + } + + /// Shows the copy feedback; a newer flash invalidates older timers, so + /// rapid line clicks don't clear each other's message early. + private func flashCopy(line: String) { + copiedFlash = String(localized: "preview.panel.copyLine") + ": " + line + flashGeneration += 1 + let generation = flashGeneration + DispatchQueue.main.asyncAfter(deadline: .now() + 1.2) { + if flashGeneration == generation { + copiedFlash = nil } } } diff --git a/TransFrame/Features/Preview/PreviewToolBar.swift b/TransFrame/Features/Preview/PreviewToolBar.swift index 1c065e8..2953cc2 100644 --- a/TransFrame/Features/Preview/PreviewToolBar.swift +++ b/TransFrame/Features/Preview/PreviewToolBar.swift @@ -40,6 +40,7 @@ struct PreviewToolBar: View { : Color.clear ) .clipShape(RoundedRectangle(cornerRadius: 4)) + .contentShape(Rectangle().inset(by: -2)) .customTooltip("\(tool.displayName) (\(String(tool.keyboardShortcut).uppercased()))") .accessibilityLabel(Text(tool.displayName)) .accessibilityHint(Text("Press \(String(tool.keyboardShortcut).uppercased()) to toggle")) @@ -62,6 +63,16 @@ struct PreviewStyleCustomizationBar: View { [.red, .yellow, .green, .blue, .black] } + private var isShapeTool: Bool { + effectiveToolType == .rectangle || effectiveToolType == .ellipse + } + + private var isShapeFilled: Bool { + isEditingAnnotation + ? (viewModel.selectedAnnotationIsFilled ?? false) + : (effectiveToolType == .ellipse ? AppSettings.shared.ellipseFilled : AppSettings.shared.rectangleFilled) + } + var body: some View { HStack(spacing: 8) { if isEditingAnnotation { @@ -71,23 +82,28 @@ struct PreviewStyleCustomizationBar: View { } if effectiveToolType != .mosaic { - colorPicker - + colorRow(for: .stroke) Divider() .frame(height: 16) } - if effectiveToolType == .rectangle || effectiveToolType == .ellipse { + if isShapeTool { shapeFillToggle Divider() .frame(height: 16) } + if isShapeTool && isShapeFilled { + colorRow(for: .fill) + Divider() + .frame(height: 16) + } + if shouldShowStrokeWidth { strokeWidthControl } - if effectiveToolType == .text { + if effectiveToolType == .text || effectiveToolType == .calloutBubble { textSizeControl } @@ -96,6 +112,12 @@ struct PreviewStyleCustomizationBar: View { } if isEditingAnnotation { + if effectiveToolType == .text || effectiveToolType == .calloutBubble { + Divider() + .frame(height: 16) + editTextButton + } + Divider() .frame(height: 16) @@ -108,70 +130,92 @@ struct PreviewStyleCustomizationBar: View { if effectiveToolType == .freehand || effectiveToolType == .arrow || effectiveToolType == .line { return true } - - if effectiveToolType == .rectangle { - let isFilled = isEditingAnnotation - ? (viewModel.selectedAnnotationIsFilled ?? false) - : AppSettings.shared.rectangleFilled - return !isFilled - } - - if effectiveToolType == .ellipse { - let isFilled = isEditingAnnotation - ? (viewModel.selectedAnnotationIsFilled ?? false) - : AppSettings.shared.ellipseFilled - return !isFilled + + if isShapeTool { + return !isShapeFilled } - + return false } - private var colorPicker: some View { + // MARK: - Color Rows + + /// Which annotation color a swatch/picker applies to + private enum ColorTarget { + case stroke + case fill + } + + private func colorRow(for target: ColorTarget) -> some View { HStack(spacing: 2) { + if target == .fill { + Image(systemName: "paintpalette") + .font(.system(size: 9)) + .foregroundStyle(.secondary) + } + ForEach(presetColors, id: \.self) { color in - colorButton(for: color) + colorButton(for: color, target: target) } ColorPicker( "", selection: Binding( - get: { - if isEditingAnnotation { - return viewModel.selectedAnnotationColor?.color ?? .red - } - return AppSettings.shared.strokeColor.color - }, + get: { currentColor(for: target)?.color ?? .red }, set: { newColor in - if isEditingAnnotation { - viewModel.updateSelectedAnnotationColor(CodableColor(newColor)) - } else { - AppSettings.shared.strokeColor = CodableColor(newColor) - } + applyColor(CodableColor(newColor), to: target) } ), - supportsOpacity: false + supportsOpacity: target == .fill ) .labelsHidden() .frame(width: 24) + .customTooltip(String(localized: "preview.color.custom")) + + eyedropperButton(target: target) } } - private func colorButton(for color: Color) -> some View { - Button { - if isEditingAnnotation { - viewModel.updateSelectedAnnotationColor(CodableColor(color)) - } else { - AppSettings.shared.strokeColor = CodableColor(color) + private func currentColor(for target: ColorTarget) -> CodableColor? { + if isEditingAnnotation { + switch target { + case .stroke: + return viewModel.selectedAnnotationColor + case .fill: + return viewModel.selectedAnnotationFillColor + } + } + return target == .stroke ? AppSettings.shared.strokeColor : AppSettings.shared.shapeFillColor + } + + private func applyColor(_ color: CodableColor, to target: ColorTarget) { + if isEditingAnnotation { + switch target { + case .stroke: + viewModel.updateSelectedAnnotationColor(color) + case .fill: + viewModel.updateSelectedAnnotationFillColor(color) + } + } else { + switch target { + case .stroke: + AppSettings.shared.strokeColor = color + case .fill: + AppSettings.shared.shapeFillColor = color } + } + } + + private func colorButton(for color: Color, target: ColorTarget) -> some View { + Button { + applyColor(CodableColor(color), to: target) } label: { Circle() .fill(color) .frame(width: 16, height: 16) .overlay { - let currentColor = isEditingAnnotation - ? (viewModel.selectedAnnotationColor?.color ?? .clear) - : AppSettings.shared.strokeColor.color - if colorsAreEqual(currentColor, color) { + let active = currentColor(for: target)?.color ?? .clear + if colorsAreEqual(active, color) { Circle() .stroke(Color.primary, lineWidth: 2) } @@ -184,7 +228,24 @@ struct PreviewStyleCustomizationBar: View { } } .buttonStyle(.plain) - .help(colorName(for: color)) + .customTooltip(target == .fill ? "\(colorName(for: color)) (\(String(localized: "preview.fill.color")))" : colorName(for: color)) + } + + /// Screen eyedropper: samples a pixel anywhere on screen into the target color + private func eyedropperButton(target: ColorTarget) -> some View { + Button { + NSColorSampler().show { selectedColor in + guard let nsColor = selectedColor else { return } + applyColor(CodableColor(Color(nsColor: nsColor)), to: target) + } + } label: { + Image(systemName: "eyedropper") + .font(.system(size: 11)) + .frame(width: 20, height: 20) + } + .buttonStyle(.plain) + .contentShape(Rectangle().inset(by: -2)) + .customTooltip(String(localized: "preview.color.eyedropper")) } private var shapeFillToggle: some View { @@ -213,7 +274,7 @@ struct PreviewStyleCustomizationBar: View { : Color.clear ) .clipShape(RoundedRectangle(cornerRadius: 4)) - .help(isFilled ? String(localized: "preview.shape.filled") : String(localized: "preview.shape.hollow")) + .customTooltip(isFilled ? String(localized: "preview.shape.filled") : String(localized: "preview.shape.hollow")) } private func getFillIconName() -> String { @@ -250,7 +311,7 @@ struct PreviewStyleCustomizationBar: View { step: 0.5 ) .frame(width: 80) - .help(String(localized: "settings.stroke.width")) + .customTooltip(String(localized: "settings.stroke.width")) let width = isEditingAnnotation ? Int(viewModel.selectedAnnotationStrokeWidth ?? 3) @@ -288,7 +349,7 @@ struct PreviewStyleCustomizationBar: View { step: 1 ) .frame(width: 80) - .help(String(localized: "settings.text.size")) + .customTooltip(String(localized: "settings.text.size")) let size = isEditingAnnotation ? Int(viewModel.selectedAnnotationFontSize ?? 16) @@ -326,7 +387,7 @@ struct PreviewStyleCustomizationBar: View { step: 1 ) .frame(width: 80) - .help(String(localized: "settings.mosaic.blockSize")) + .customTooltip(String(localized: "settings.mosaic.blockSize")) let size = isEditingAnnotation ? Int(viewModel.selectedAnnotationBlockSize ?? 10) @@ -338,6 +399,17 @@ struct PreviewStyleCustomizationBar: View { } } + private var editTextButton: some View { + Button { + viewModel.editSelectedText() + } label: { + Image(systemName: "character.cursor.ibeam") + .frame(width: 20, height: 20) + } + .buttonStyle(.plain) + .customTooltip(String(localized: "preview.tooltip.editText")) + } + private var deleteButton: some View { Button { viewModel.deleteSelectedAnnotation() @@ -346,7 +418,7 @@ struct PreviewStyleCustomizationBar: View { .foregroundStyle(.red) } .buttonStyle(.plain) - .help(String(localized: "preview.tooltip.delete")) + .customTooltip(String(localized: "preview.tooltip.delete")) } private func colorsAreEqual(_ colorA: Color, _ colorB: Color) -> Bool { @@ -377,25 +449,30 @@ struct PreviewStyleCustomizationBar: View { // MARK: - Custom Tooltip for Floating Windows +enum TooltipPlacement { + case top + case bottom +} + extension View { - /// Custom tooltip that works in floating windows where .help() may not display - func customTooltip(_ text: String) -> some View { - self.modifier(CustomTooltipModifier(text: text)) + /// Custom tooltip that works reliably in high-level borderless floating panels where native .help() may not display + func customTooltip(_ text: String, placement: TooltipPlacement = .top) -> some View { + self.modifier(CustomTooltipModifier(text: text, placement: placement)) } } struct CustomTooltipModifier: ViewModifier { let text: String + var placement: TooltipPlacement = .top @State private var isHovered = false @State private var showTooltip = false - + func body(content: Content) -> some View { content .onHover { hovering in isHovered = hovering if hovering { - // Short delay before showing tooltip - DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.25) { if isHovered { showTooltip = true } @@ -404,21 +481,29 @@ struct CustomTooltipModifier: ViewModifier { showTooltip = false } } - .overlay(alignment: .top) { + .overlay(alignment: placement == .top ? .top : .bottom) { if showTooltip { Text(text) - .font(.caption) + .font(.system(size: 11, weight: .medium)) .padding(.horizontal, 6) .padding(.vertical, 3) - .background(Color(nsColor: .windowBackgroundColor)) + .background( + RoundedRectangle(cornerRadius: 4) + .fill(Color(nsColor: .windowBackgroundColor)) + .shadow(color: .black.opacity(0.25), radius: 3, x: 0, y: 1) + ) + .overlay( + RoundedRectangle(cornerRadius: 4) + .stroke(Color.primary.opacity(0.15), lineWidth: 0.5) + ) .foregroundColor(.primary) - .cornerRadius(4) - .shadow(color: .black.opacity(0.2), radius: 2, x: 0, y: 1) - .offset(y: -28) + .offset(y: placement == .top ? -28 : 28) .transition(.opacity) .fixedSize() - .compositingGroup() + .allowsHitTesting(false) + .zIndex(999) } } + .zIndex(showTooltip ? 100 : 0) } } diff --git a/TransFrame/Features/Preview/PreviewViewModel+Crop.swift b/TransFrame/Features/Preview/PreviewViewModel+Crop.swift index f46d0b2..e073701 100644 --- a/TransFrame/Features/Preview/PreviewViewModel+Crop.swift +++ b/TransFrame/Features/Preview/PreviewViewModel+Crop.swift @@ -63,60 +63,8 @@ extension PreviewViewModel { // Shift annotations relative to the new crop origin let originShift = clampedRect.origin - var updatedAnnotations: [Annotation] = [] - for anno in screenshot.annotations { - var shifted = anno - switch shifted { - case .rectangle(var r): - r.rect.origin.x -= originShift.x - r.rect.origin.y -= originShift.y - updatedAnnotations.append(.rectangle(r)) - case .ellipse(var e): - e.rect.origin.x -= originShift.x - e.rect.origin.y -= originShift.y - updatedAnnotations.append(.ellipse(e)) - case .line(var l): - l.startPoint.x -= originShift.x - l.startPoint.y -= originShift.y - l.endPoint.x -= originShift.x - l.endPoint.y -= originShift.y - updatedAnnotations.append(.line(l)) - case .freehand(var f): - f.points = f.points.map { CGPoint(x: $0.x - originShift.x, y: $0.y - originShift.y) } - updatedAnnotations.append(.freehand(f)) - case .arrow(var a): - a.startPoint.x -= originShift.x - a.startPoint.y -= originShift.y - a.endPoint.x -= originShift.x - a.endPoint.y -= originShift.y - updatedAnnotations.append(.arrow(a)) - case .highlight(var h): - h.rect.origin.x -= originShift.x - h.rect.origin.y -= originShift.y - updatedAnnotations.append(.highlight(h)) - case .mosaic(var m): - m.rect.origin.x -= originShift.x - m.rect.origin.y -= originShift.y - updatedAnnotations.append(.mosaic(m)) - case .text(var t): - t.position.x -= originShift.x - t.position.y -= originShift.y - updatedAnnotations.append(.text(t)) - case .numberLabel(var n): - n.position.x -= originShift.x - n.position.y -= originShift.y - updatedAnnotations.append(.numberLabel(n)) - case .privacyRedaction(var pr): - pr.rect.origin.x -= originShift.x - pr.rect.origin.y -= originShift.y - updatedAnnotations.append(.privacyRedaction(pr)) - case .calloutBubble(var cb): - cb.rect.origin.x -= originShift.x - cb.rect.origin.y -= originShift.y - cb.targetPoint.x -= originShift.x - cb.targetPoint.y -= originShift.y - updatedAnnotations.append(.calloutBubble(cb)) - } + let updatedAnnotations = screenshot.annotations.map { + $0.offsetBy(dx: -originShift.x, dy: -originShift.y) } // Invalidate OCR cache for the previous crop area diff --git a/TransFrame/Features/Preview/PreviewViewModel+Drawing.swift b/TransFrame/Features/Preview/PreviewViewModel+Drawing.swift index 28a6f80..bb747cd 100644 --- a/TransFrame/Features/Preview/PreviewViewModel+Drawing.swift +++ b/TransFrame/Features/Preview/PreviewViewModel+Drawing.swift @@ -9,8 +9,26 @@ extension PreviewViewModel { /// Begins a drawing gesture at the given point /// - Parameter point: The point in image coordinates func beginDrawing(at point: CGPoint) { + if isWaitingForTextInputInternal { + commitTextInput() + } + guard let selectedTool else { return } + // If user clicks on an existing text or callout bubble while using text/callout tool, edit it in-place + if selectedTool == .text || selectedTool == .calloutBubble, + let hitIdx = hitTest(at: point) { + switch annotations[hitIdx] { + case .text, .calloutBubble: + beginEditingTextAnnotation(at: hitIdx) + return + default: + break + } + } + + let scaleFactor = screenshot.sourceDisplay.scaleFactor + // Apply current stroke/text styles from settings let strokeStyle = StrokeStyle( color: settings.strokeColor, @@ -26,10 +44,12 @@ extension PreviewViewModel { case .rectangle: rectangleTool.strokeStyle = strokeStyle rectangleTool.isFilled = settings.rectangleFilled + rectangleTool.fillColor = settings.shapeFillColor rectangleTool.beginDrawing(at: point) case .ellipse: ellipseTool.strokeStyle = strokeStyle ellipseTool.isFilled = settings.ellipseFilled + ellipseTool.fillColor = settings.shapeFillColor ellipseTool.beginDrawing(at: point) case .line: lineTool.strokeStyle = strokeStyle @@ -41,19 +61,23 @@ extension PreviewViewModel { arrowTool.strokeStyle = strokeStyle arrowTool.beginDrawing(at: point) case .highlight: + highlightTool.highlightColor = settings.strokeColor highlightTool.beginDrawing(at: point) case .mosaic: mosaicTool.blockSize = Int(settings.mosaicBlockSize) mosaicTool.beginDrawing(at: point) case .text: + editingAnnotationIndex = nil textTool.textStyle = textStyle textTool.beginDrawing(at: point) isWaitingForTextInputInternal = true textInputPositionInternal = point case .numberLabel: + numberLabelTool.labelColor = settings.strokeColor + numberLabelTool.circleSize = 24 * scaleFactor numberLabelTool.beginDrawing(at: point) case .privacyRedaction: - rectangleTool.strokeStyle = strokeStyle + rectangleTool.strokeStyle = StrokeStyle(color: CodableColor(.black), lineWidth: strokeStyle.lineWidth) rectangleTool.isFilled = true rectangleTool.beginDrawing(at: point) case .calloutBubble: @@ -102,6 +126,7 @@ extension PreviewViewModel { guard let selectedTool else { return } var annotation: Annotation? + let scaleFactor = screenshot.sourceDisplay.scaleFactor switch selectedTool { case .rectangle: @@ -129,10 +154,34 @@ extension PreviewViewModel { annotation = .privacyRedaction(PrivacyRedactionAnnotation(rect: r.rect, fillColor: CodableColor(.black))) } case .calloutBubble: - if let rectAnno = rectangleTool.endDrawing(at: point), case .rectangle(let r) = rectAnno { - let target = CGPoint(x: r.rect.midX, y: r.rect.maxY + 20) - annotation = .calloutBubble(CalloutBubbleAnnotation(rect: r.rect, targetPoint: target, text: "Note")) + let rawAnno = rectangleTool.endDrawing(at: point) + let bubbleRect: CGRect + if let rawAnno, case .rectangle(let r) = rawAnno, r.rect.width >= 30 * scaleFactor, r.rect.height >= 24 * scaleFactor { + bubbleRect = r.rect + } else { + let defaultW = 120 * scaleFactor + let defaultH = 48 * scaleFactor + bubbleRect = CGRect(x: point.x, y: point.y, width: defaultW, height: defaultH) } + let target = CGPoint(x: bubbleRect.midX, y: bubbleRect.maxY + 20 * scaleFactor) + let textStyle = TextStyle( + color: CodableColor(.black), + fontSize: settings.textSize, + fontName: ".AppleSystemUIFont" + ) + let bubble = CalloutBubbleAnnotation( + rect: bubbleRect, + targetPoint: target, + text: "Note", + style: textStyle, + bubbleColor: CodableColor(.white) + ) + currentAnnotationInternal = nil + drawingUpdateCounter += 1 + addAnnotation(.calloutBubble(bubble)) + let newIdx = annotations.count - 1 + beginEditingTextAnnotation(at: newIdx) + return } currentAnnotationInternal = nil @@ -143,7 +192,7 @@ extension PreviewViewModel { } } - /// Cancels the current drawing operation + /// Cancels the current drawing or in-place text editing operation without losing existing annotations func cancelCurrentDrawing() { rectangleTool.cancelDrawing() ellipseTool.cancelDrawing() @@ -157,6 +206,7 @@ extension PreviewViewModel { currentAnnotationInternal = nil isWaitingForTextInputInternal = false textInputPositionInternal = nil + editingAnnotationIndex = nil drawingUpdateCounter += 1 } @@ -166,13 +216,62 @@ extension PreviewViewModel { drawingUpdateCounter += 1 } - /// Commits the current text input and adds the annotation + /// Commits the current text input and adds or updates the annotation func commitTextInput() { - guard let position = textInputPositionInternal else { return } + guard isWaitingForTextInputInternal else { return } let trimmedText = textInputContent.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedText.isEmpty else { - cancelCurrentDrawing() + + // Reset observable text input state first to prevent re-entrancy + let editIdx = editingAnnotationIndex + let position = textInputPositionInternal + textInputContent = "" + isWaitingForTextInputInternal = false + textInputPositionInternal = nil + editingAnnotationIndex = nil + textTool.cancelDrawing() + + // Case 1: Updating an existing text or calloutBubble annotation in-place (preserving Z-Order and Undo history) + if let editIdx, editIdx >= 0, editIdx < annotations.count { + switch annotations[editIdx] { + case .text(var textAnno): + if trimmedText.isEmpty { + // Removing the annotation shifts later indices down; + // re-selecting editIdx would grab an unrelated neighbor. + removeAnnotation(at: editIdx) + if selectedAnnotationIndex == editIdx { + selectedAnnotationIndex = nil + } + } else { + if textAnno.content != trimmedText { + pushUndoState() + textAnno.content = trimmedText + screenshot = screenshot.replacingAnnotation(at: editIdx, with: .text(textAnno)) + redoStack.removeAll() + } + // Keep the text box selected so its handles show for drag/resize + if editIdx < annotations.count { + selectedAnnotationIndex = editIdx + } + } + case .calloutBubble(var bubbleAnno): + let finalText = trimmedText.isEmpty ? bubbleAnno.text : trimmedText + if bubbleAnno.text != finalText { + pushUndoState() + bubbleAnno.text = finalText + screenshot = screenshot.replacingAnnotation(at: editIdx, with: .calloutBubble(bubbleAnno)) + redoStack.removeAll() + } + default: + break + } + drawingUpdateCounter += 1 + return + } + + // Case 2: Creating a new text annotation + guard let position, !trimmedText.isEmpty else { + drawingUpdateCounter += 1 return } @@ -185,25 +284,29 @@ extension PreviewViewModel { TextAnnotation(position: position, content: trimmedText, style: textStyle) ) addAnnotation(annotation) - - // Reset observable text input state - textInputContent = "" - isWaitingForTextInputInternal = false - textInputPositionInternal = nil + // Select the new text box so its handles show for drag/resize (PRD 5.2) + selectedAnnotationIndex = annotations.count - 1 + drawingUpdateCounter += 1 } - // MARK: - Annotation Selection & Editing + // MARK: - Annotation Selection, Resizing & Editing + + /// Tests if a point hits a control handle of the currently selected annotation + func hitTestSelectedAnnotationHandle(at point: CGPoint) -> AnnotationHandle? { + guard let index = selectedAnnotationIndex, + index >= 0, + index < annotations.count else { return nil } + let tolerance = 10.0 * screenshot.sourceDisplay.scaleFactor + return annotations[index].hitTestHandle(at: point, padding: 4, tolerance: tolerance) + } - /// Tests if a point hits an annotation and returns its index + /// Tests if a point hits an annotation using exact geometric distance and returns its index /// - Parameter point: The point to test in image coordinates /// - Returns: The index of the hit annotation, or nil if none hit func hitTest(at point: CGPoint) -> Int? { - // Check in reverse order (top-most first) + let tolerance = 8.0 * screenshot.sourceDisplay.scaleFactor for (index, annotation) in annotations.enumerated().reversed() { - let bounds = annotation.bounds - // Add some padding for easier selection - let expandedBounds = bounds.insetBy(dx: -10, dy: -10) - if expandedBounds.contains(point) { + if annotation.hitTest(point: point, tolerance: tolerance) { return index } } @@ -212,6 +315,9 @@ extension PreviewViewModel { /// Selects the annotation at the given index func selectAnnotation(at index: Int?) { + if isWaitingForTextInputInternal && editingAnnotationIndex != index { + commitTextInput() + } // Deselect any tool when selecting an annotation if index != nil && selectedTool != nil { selectedTool = nil @@ -221,55 +327,92 @@ extension PreviewViewModel { /// Deselects any selected annotation func deselectAnnotation() { + if isWaitingForTextInputInternal { + commitTextInput() + } selectedAnnotationIndex = nil isDraggingAnnotation = false + activeResizeHandle = nil dragStartPoint = nil dragOriginalPosition = nil + dragOriginalAnnotation = nil + dragInitialScreenshot = nil } /// Deletes the currently selected annotation func deleteSelectedAnnotation() { guard let index = selectedAnnotationIndex else { return } + if editingAnnotationIndex == index { + cancelCurrentDrawing() + } pushUndoState() screenshot = screenshot.removingAnnotation(at: index) redoStack.removeAll() selectedAnnotationIndex = nil } + /// Begins resizing or adjusting a control handle on the selected annotation + func beginResizingAnnotation(handle: AnnotationHandle, at point: CGPoint) { + guard let index = selectedAnnotationIndex, + index >= 0, + index < annotations.count else { return } + + if isWaitingForTextInputInternal { + commitTextInput() + } + + activeResizeHandle = handle + dragStartPoint = point + dragOriginalAnnotation = annotations[index] + dragInitialScreenshot = screenshot + } + + /// Continues resizing or adjusting a control handle on the selected annotation + func continueResizingAnnotation(to point: CGPoint) { + guard let handle = activeResizeHandle, + let index = selectedAnnotationIndex, + index >= 0, + index < annotations.count else { return } + + // ⌘ held: preserve the annotation's current aspect ratio + let options = HandleDragOptions( + preserveAspectRatio: NSEvent.modifierFlags.contains(.command) + ) + let updated = annotations[index].applyingHandleDrag(handle, to: point, options: options) + screenshot.annotations[index] = updated + drawingUpdateCounter += 1 + } + + /// Ends resizing or adjusting a control handle and records Undo if modified + func endResizingAnnotation() { + guard activeResizeHandle != nil else { return } + if let initial = dragInitialScreenshot, initial.annotations != screenshot.annotations { + undoStack.append(initial) + if undoStack.count > maxUndoLevels { + undoStack.removeFirst() + } + redoStack.removeAll() + } + activeResizeHandle = nil + dragStartPoint = nil + dragOriginalAnnotation = nil + dragInitialScreenshot = nil + } + /// Begins dragging the selected annotation func beginDraggingAnnotation(at point: CGPoint) { guard let index = selectedAnnotationIndex, index < annotations.count else { return } + if isWaitingForTextInputInternal && editingAnnotationIndex != index { + commitTextInput() + } + isDraggingAnnotation = true dragStartPoint = point - - // Store the original position based on annotation type - let annotation = annotations[index] - switch annotation { - case .rectangle(let rect): - dragOriginalPosition = rect.rect.origin - case .ellipse(let ellipse): - dragOriginalPosition = ellipse.rect.origin - case .line(let line): - dragOriginalPosition = line.bounds.origin - case .freehand(let freehand): - dragOriginalPosition = freehand.bounds.origin - case .arrow(let arrow): - dragOriginalPosition = arrow.bounds.origin - case .highlight(let highlight): - dragOriginalPosition = highlight.rect.origin - case .mosaic(let mosaic): - dragOriginalPosition = mosaic.rect.origin - case .text(let text): - dragOriginalPosition = text.position - case .numberLabel(let label): - dragOriginalPosition = label.position - case .privacyRedaction(let redaction): - dragOriginalPosition = redaction.rect.origin - case .calloutBubble(let bubble): - dragOriginalPosition = bubble.rect.origin - } + dragOriginalAnnotation = annotations[index] + dragInitialScreenshot = screenshot + dragOriginalPosition = annotations[index].bounds.origin } /// Continues dragging the selected annotation @@ -277,7 +420,7 @@ extension PreviewViewModel { guard isDraggingAnnotation, let index = selectedAnnotationIndex, let startPoint = dragStartPoint, - let originalPosition = dragOriginalPosition, + let originalAnnotation = dragOriginalAnnotation, index < annotations.count else { return } let delta = CGPoint( @@ -285,116 +428,27 @@ extension PreviewViewModel { y: point.y - startPoint.y ) - let annotation = annotations[index] - var updatedAnnotation: Annotation? - - switch annotation { - case .rectangle(var rect): - rect.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .rectangle(rect) - - case .ellipse(var ellipse): - ellipse.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .ellipse(ellipse) - - case .line(var line): - let bounds = line.bounds - let offsetX = originalPosition.x + delta.x - bounds.origin.x - let offsetY = originalPosition.y + delta.y - bounds.origin.y - line.startPoint = CGPoint(x: line.startPoint.x + offsetX, y: line.startPoint.y + offsetY) - line.endPoint = CGPoint(x: line.endPoint.x + offsetX, y: line.endPoint.y + offsetY) - updatedAnnotation = .line(line) - - case .freehand(var freehand): - let bounds = freehand.bounds - let offsetX = originalPosition.x + delta.x - bounds.origin.x - let offsetY = originalPosition.y + delta.y - bounds.origin.y - freehand.points = freehand.points.map { point in - CGPoint(x: point.x + offsetX, y: point.y + offsetY) - } - updatedAnnotation = .freehand(freehand) - - case .arrow(var arrow): - let bounds = arrow.bounds - let offsetX = originalPosition.x + delta.x - bounds.origin.x - let offsetY = originalPosition.y + delta.y - bounds.origin.y - arrow.startPoint = CGPoint( - x: arrow.startPoint.x + offsetX, - y: arrow.startPoint.y + offsetY - ) - arrow.endPoint = CGPoint( - x: arrow.endPoint.x + offsetX, - y: arrow.endPoint.y + offsetY - ) - updatedAnnotation = .arrow(arrow) - - case .highlight(var highlight): - highlight.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .highlight(highlight) - - case .mosaic(var mosaic): - mosaic.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .mosaic(mosaic) - - case .text(var text): - text.position = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .text(text) - - case .numberLabel(var label): - label.position = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .numberLabel(label) - - case .privacyRedaction(var redaction): - redaction.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - updatedAnnotation = .privacyRedaction(redaction) - - case .calloutBubble(var bubble): - let shiftX = originalPosition.x + delta.x - bubble.rect.origin.x - let shiftY = originalPosition.y + delta.y - bubble.rect.origin.y - bubble.rect.origin = CGPoint( - x: originalPosition.x + delta.x, - y: originalPosition.y + delta.y - ) - bubble.targetPoint = CGPoint( - x: bubble.targetPoint.x + shiftX, - y: bubble.targetPoint.y + shiftY - ) - updatedAnnotation = .calloutBubble(bubble) - } - - if let updated = updatedAnnotation { - // Update without pushing undo (will push on end) - screenshot.annotations[index] = updated - drawingUpdateCounter += 1 - } + let updated = originalAnnotation.offsetBy(dx: delta.x, dy: delta.y) + screenshot.annotations[index] = updated + drawingUpdateCounter += 1 } - /// Ends dragging the selected annotation + /// Ends dragging the selected annotation and records Undo state if moved func endDraggingAnnotation() { + if isDraggingAnnotation, + let initial = dragInitialScreenshot, + initial.annotations != screenshot.annotations { + undoStack.append(initial) + if undoStack.count > maxUndoLevels { + undoStack.removeFirst() + } + redoStack.removeAll() + } isDraggingAnnotation = false dragStartPoint = nil dragOriginalPosition = nil + dragOriginalAnnotation = nil + dragInitialScreenshot = nil } /// Updates the color of the selected annotation @@ -497,18 +551,28 @@ extension PreviewViewModel { } } - /// Updates the font size of the selected text annotation + /// Updates the font size of the selected text or callout bubble annotation func updateSelectedAnnotationFontSize(_ size: CGFloat) { guard let index = selectedAnnotationIndex, index < annotations.count else { return } let annotation = annotations[index] - guard case .text(var text) = annotation else { return } - - pushUndoState() - text.style.fontSize = size - screenshot = screenshot.replacingAnnotation(at: index, with: .text(text)) - redoStack.removeAll() + switch annotation { + case .text(var text): + guard text.style.fontSize != size else { return } + pushUndoState() + text.style.fontSize = size + screenshot = screenshot.replacingAnnotation(at: index, with: .text(text)) + redoStack.removeAll() + case .calloutBubble(var bubble): + guard bubble.style.fontSize != size else { return } + pushUndoState() + bubble.style.fontSize = size + screenshot = screenshot.replacingAnnotation(at: index, with: .calloutBubble(bubble)) + redoStack.removeAll() + default: + return + } } /// Updates the isFilled state of the selected rectangle or ellipse annotation @@ -517,8 +581,7 @@ extension PreviewViewModel { index < annotations.count else { return } let annotation = annotations[index] - - // Check if the fill state is actually changing + let shouldUpdate: Bool if case .rectangle(let rect) = annotation { shouldUpdate = rect.isFilled != isFilled @@ -527,11 +590,11 @@ extension PreviewViewModel { } else { return } - + guard shouldUpdate else { return } - + pushUndoState() - + if case .rectangle(var rect) = annotation { rect.isFilled = isFilled screenshot = screenshot.replacingAnnotation(at: index, with: .rectangle(rect)) @@ -539,10 +602,33 @@ extension PreviewViewModel { ellipse.isFilled = isFilled screenshot = screenshot.replacingAnnotation(at: index, with: .ellipse(ellipse)) } - + redoStack.removeAll() } + /// Updates the background fill color of the selected rectangle or ellipse annotation + func updateSelectedAnnotationFillColor(_ color: CodableColor) { + guard let index = selectedAnnotationIndex, + index < annotations.count else { return } + + switch annotations[index] { + case .rectangle(var rect): + guard rect.fillColor != color else { return } + pushUndoState() + rect.fillColor = color + screenshot = screenshot.replacingAnnotation(at: index, with: .rectangle(rect)) + redoStack.removeAll() + case .ellipse(var ellipse): + guard ellipse.fillColor != color else { return } + pushUndoState() + ellipse.fillColor = color + screenshot = screenshot.replacingAnnotation(at: index, with: .ellipse(ellipse)) + redoStack.removeAll() + default: + return + } + } + /// Updates the block size of the selected mosaic annotation func updateSelectedAnnotationBlockSize(_ blockSize: Int) { guard let index = selectedAnnotationIndex, @@ -551,7 +637,6 @@ extension PreviewViewModel { let annotation = annotations[index] guard case .mosaic(var mosaic) = annotation else { return } - // Enforce minimum block size of 1 to prevent rendering errors let clampedBlockSize = max(1, blockSize) guard mosaic.blockSize != clampedBlockSize else { return } @@ -560,4 +645,39 @@ extension PreviewViewModel { screenshot = screenshot.replacingAnnotation(at: index, with: .mosaic(mosaic)) redoStack.removeAll() } + + /// Re-edits the selected text or calloutBubble annotation in-place without deleting or altering Z-Order + func editSelectedText() { + guard let index = selectedAnnotationIndex, index < annotations.count else { return } + beginEditingTextAnnotation(at: index) + } + + /// Begins in-place editing of a text or calloutBubble annotation at the given index + func beginEditingTextAnnotation(at index: Int) { + guard index >= 0 && index < annotations.count else { return } + if isWaitingForTextInputInternal && editingAnnotationIndex != index { + commitTextInput() + } + + switch annotations[index] { + case .text(let text): + selectedAnnotationIndex = index + editingAnnotationIndex = index + textInputContent = text.content + textInputPositionInternal = text.position + isWaitingForTextInputInternal = true + case .calloutBubble(let bubble): + selectedAnnotationIndex = index + editingAnnotationIndex = index + textInputContent = bubble.text + let scaleFactor = screenshot.sourceDisplay.scaleFactor + textInputPositionInternal = CGPoint( + x: bubble.rect.minX + 6 * scaleFactor, + y: bubble.rect.minY + 6 * scaleFactor + ) + isWaitingForTextInputInternal = true + default: + break + } + } } diff --git a/TransFrame/Features/Preview/PreviewViewModel+Export.swift b/TransFrame/Features/Preview/PreviewViewModel+Export.swift index 9a4844e..4940dd5 100644 --- a/TransFrame/Features/Preview/PreviewViewModel+Export.swift +++ b/TransFrame/Features/Preview/PreviewViewModel+Export.swift @@ -5,6 +5,9 @@ import AppKit extension PreviewViewModel { @discardableResult func copyToClipboard(dismissOnSuccess: Bool = false) -> Bool { + if isWaitingForTextInputInternal { + commitTextInput() + } guard !isCopying else { return false } isCopying = true @@ -37,6 +40,9 @@ extension PreviewViewModel { } func saveScreenshot() { + if isWaitingForTextInputInternal { + commitTextInput() + } guard !isSaving else { return } isSaving = true diff --git a/TransFrame/Features/Preview/PreviewViewModel+OCR.swift b/TransFrame/Features/Preview/PreviewViewModel+OCR.swift index 84e2d3a..06c189e 100644 --- a/TransFrame/Features/Preview/PreviewViewModel+OCR.swift +++ b/TransFrame/Features/Preview/PreviewViewModel+OCR.swift @@ -4,7 +4,11 @@ import AppKit extension PreviewViewModel { func performOCR() { + if isWaitingForTextInputInternal { + commitTextInput() + } guard !isPerformingOCR else { return } + resultsPanelTab = .ocr isPerformingOCR = true ocrTranslationError = nil @@ -16,10 +20,17 @@ extension PreviewViewModel { func executeOCR() async { defer { isPerformingOCR = false } + // Recognize in the user's source language; auto (or an unsupported + // language) falls back to Vision's automatic language detection. + let languages = Set( + [AppSettings.shared.translationSourceLanguage] + .compactMap { OCREngine.RecognitionLanguage(translationLanguage: $0) } + ) + do { let result = try await ocrService.recognize( image, - languages: [.english, .chineseSimplified] + languages: languages ) ocrResult = result if result.observations.isEmpty { diff --git a/TransFrame/Features/Preview/PreviewViewModel+SmartRedact.swift b/TransFrame/Features/Preview/PreviewViewModel+SmartRedact.swift new file mode 100644 index 0000000..2613dc9 --- /dev/null +++ b/TransFrame/Features/Preview/PreviewViewModel+SmartRedact.swift @@ -0,0 +1,68 @@ +import Foundation +import SwiftUI +import os + +// MARK: - Smart Privacy Redaction (智能马赛克) + +extension PreviewViewModel { + private static let smartRedactLogger = Logger.ui + + /// Runs VLM privacy detection over the current image and overlays mosaic + /// annotations on every detected sensitive region. One undo step covers the batch. + func performSmartRedaction() { + guard !isDetectingPrivacy else { return } + if isWaitingForTextInputInternal { + commitTextInput() + } + + isDetectingPrivacy = true + smartRedactionMessage = String(localized: "smartMosaic.detecting") + + Task { @MainActor in + defer { isDetectingPrivacy = false } + + do { + let rects = try await SmartPrivacyRedactionService.shared.detectPrivacyRegions(in: image) + + guard !rects.isEmpty else { + smartRedactionMessage = String(localized: "smartMosaic.noneFound") + dismissSmartRedactionMessage() + return + } + + let blockSize = Int(settings.mosaicBlockSize) + let mosaics = rects.map { rect in + Annotation.mosaic(MosaicAnnotation(rect: rect, blockSize: blockSize)) + } + + pushUndoState() + for mosaic in mosaics { + screenshot = screenshot.adding(mosaic) + } + redoStack.removeAll() + drawingUpdateCounter += 1 + + smartRedactionMessage = String(format: String(localized: "smartMosaic.redacted"), rects.count) + dismissSmartRedactionMessage() + } catch { + Self.smartRedactLogger.error("Smart redaction failed: \(error.localizedDescription)") + smartRedactionMessage = nil + errorMessage = String(localized: "smartMosaic.error.failed") + clearError() + } + } + } + + /// Auto-hides the transient smart-redaction toast. + /// Captures the message it was scheduled for: if a newer detection has + /// replaced it meanwhile, the stale task leaves the new toast alone. + func dismissSmartRedactionMessage() { + let scheduledFor = smartRedactionMessage + Task { + try? await Task.sleep(for: .seconds(3)) + if smartRedactionMessage == scheduledFor { + smartRedactionMessage = nil + } + } + } +} diff --git a/TransFrame/Features/Preview/PreviewViewModel+Translation.swift b/TransFrame/Features/Preview/PreviewViewModel+Translation.swift new file mode 100644 index 0000000..47c5c6a --- /dev/null +++ b/TransFrame/Features/Preview/PreviewViewModel+Translation.swift @@ -0,0 +1,64 @@ +import Foundation +import SwiftUI + +// MARK: - Embedded Translation (预览窗口内嵌翻译) + +extension PreviewViewModel { + var hasTranslationResults: Bool { + !bilingualSegments.isEmpty + } + + /// Runs the shared image translation pipeline (analyze + translate) and + /// stores bilingual segments for the embedded side panel. No external window. + func performTranslation() { + guard !isTranslating else { return } + if isWaitingForTextInputInternal { + commitTextInput() + } + + resultsPanelTab = .translate + isTranslating = true + ocrTranslationError = nil + + Task { @MainActor in + defer { isTranslating = false } + + let pipeline = ImageTranslationPipeline() + let context = PipelineContext() + + // Stage-split so failures are attributed correctly: analysis + // (VLM) errors read as recognition, not translation failures. + let analysis: ScreenAnalysisResult + do { + analysis = try await pipeline.analyze(image: image, context: context) + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + errorMessage = String(format: String(localized: "error.pipeline.ocrFailed"), message) + ocrTranslationError = errorMessage + clearError() + return + } + + let filtered = analysis.filteredForTranslation() + guard !filtered.segments.isEmpty else { + errorMessage = String(localized: "error.ocr.no.text") + clearError() + return + } + + let config = ImageTranslationPipelineConfig.fromAppSettings(scene: nil) + do { + bilingualSegments = try await pipeline.translate( + analysisResult: filtered, + config: config, + context: context + ) + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + errorMessage = String(format: String(localized: "error.pipeline.translateFailed"), message) + ocrTranslationError = errorMessage + clearError() + } + } + } +} diff --git a/TransFrame/Features/Preview/PreviewViewModel.swift b/TransFrame/Features/Preview/PreviewViewModel.swift index 0ef1640..72d3391 100644 --- a/TransFrame/Features/Preview/PreviewViewModel.swift +++ b/TransFrame/Features/Preview/PreviewViewModel.swift @@ -19,7 +19,11 @@ final class PreviewViewModel { var selectedTool: AnnotationToolType? { didSet { if oldValue != selectedTool { - cancelCurrentDrawing() + if isWaitingForTextInputInternal { + commitTextInput() + } else { + cancelCurrentDrawing() + } } if selectedTool != nil && isCropMode { isCropMode = false @@ -30,6 +34,9 @@ final class PreviewViewModel { var isCropMode: Bool = false { didSet { + if isCropMode && isWaitingForTextInputInternal { + commitTextInput() + } if isCropMode && selectedTool != nil { selectedTool = nil } @@ -74,8 +81,20 @@ final class PreviewViewModel { var ocrResult: OCRResult? var isPerformingOCR: Bool = false + /// 面板显示哪个结果,由工具栏最后一次触发的动作决定(面板内无切换)。 + enum ResultsPanelTab { + case ocr + case translate + } + var resultsPanelTab: ResultsPanelTab = .ocr var ocrTranslationError: String? + var isDetectingPrivacy: Bool = false + var smartRedactionMessage: String? + + var bilingualSegments: [BilingualSegment] = [] + var isTranslating: Bool = false + // MARK: - Annotation Tools @ObservationIgnored @@ -113,7 +132,13 @@ final class PreviewViewModel { // MARK: - Annotation Selection & Editing var selectedAnnotationIndex: Int? + var editingAnnotationIndex: Int? var isDraggingAnnotation: Bool = false + var activeResizeHandle: AnnotationHandle? + + var isResizingAnnotation: Bool { + activeResizeHandle != nil + } @ObservationIgnored var dragStartPoint: CGPoint? @@ -121,6 +146,12 @@ final class PreviewViewModel { @ObservationIgnored var dragOriginalPosition: CGPoint? + @ObservationIgnored + var dragOriginalAnnotation: Annotation? + + @ObservationIgnored + var dragInitialScreenshot: Screenshot? + var isDrawing: Bool { currentTool?.isActive ?? false } @@ -214,7 +245,7 @@ final class PreviewViewModel { var redoStack: [Screenshot] = [] @ObservationIgnored - private let maxUndoLevels = 15 + let maxUndoLevels = 15 var imageSizeChangeCounter: Int = 0 @@ -396,6 +427,19 @@ final class PreviewViewModel { return nil } + var selectedAnnotationFillColor: CodableColor? { + guard let index = selectedAnnotationIndex, + index < annotations.count else { return nil } + + if case .rectangle(let rect) = annotations[index] { + return rect.effectiveFillColor + } + if case .ellipse(let ellipse) = annotations[index] { + return ellipse.effectiveFillColor + } + return nil + } + var selectedAnnotationBlockSize: Int? { guard let index = selectedAnnotationIndex, index < annotations.count else { return nil } @@ -418,6 +462,9 @@ final class PreviewViewModel { /// Pins the current screenshot with all annotations func pinScreenshot() { + if isWaitingForTextInputInternal { + commitTextInput() + } PinnedWindowsManager.shared.pinScreenshot(screenshot, annotations: annotations) } @@ -471,7 +518,7 @@ enum AnnotationToolType: String, CaseIterable, Identifiable, Sendable { case .mosaic: return "m" case .text: return "t" case .numberLabel: return "n" - case .privacyRedaction: return "p" + case .privacyRedaction: return "x" case .calloutBubble: return "b" } } @@ -491,4 +538,23 @@ enum AnnotationToolType: String, CaseIterable, Identifiable, Sendable { case .calloutBubble: return "bubble.left.and.bubble.right" } } + + var cursor: NSCursor { + switch self { + case .text: + return .iBeam + case .freehand: + return NSCursor.symbolCursor(systemName: "pencil", hotSpot: NSPoint(x: 3, y: 17)) ?? .crosshair + case .highlight: + return NSCursor.symbolCursor(systemName: "highlighter", hotSpot: NSPoint(x: 3, y: 17)) ?? .crosshair + case .mosaic: + return NSCursor.symbolCursor(systemName: "checkerboard.rectangle", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .privacyRedaction: + return NSCursor.symbolCursor(systemName: "eye.slash.fill", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .numberLabel: + return NSCursor.symbolCursor(systemName: "number.circle.fill", hotSpot: NSPoint(x: 10, y: 10)) ?? .crosshair + case .rectangle, .ellipse, .line, .arrow, .calloutBubble: + return .crosshair + } + } } diff --git a/TransFrame/Features/Preview/PreviewWindow.swift b/TransFrame/Features/Preview/PreviewWindow.swift index 026fe05..d5e5baf 100644 --- a/TransFrame/Features/Preview/PreviewWindow.swift +++ b/TransFrame/Features/Preview/PreviewWindow.swift @@ -47,7 +47,7 @@ final class PreviewWindow: NSPanel { super.init( contentRect: contentRect, - styleMask: [.titled, .closable, .resizable, .utilityWindow, .nonactivatingPanel], + styleMask: [.borderless, .resizable], backing: .buffered, defer: false ) @@ -64,14 +64,12 @@ final class PreviewWindow: NSPanel { // Window behavior title = NSLocalizedString("preview.window.title", comment: "Screenshot Preview") level = .floating - isMovableByWindowBackground = false + isMovableByWindowBackground = true hidesOnDeactivate = false isReleasedWhenClosed = false - - // Appearance - backgroundColor = .windowBackgroundColor - titlebarAppearsTransparent = false - titleVisibility = .visible + isOpaque = false + backgroundColor = .clear + hasShadow = true // Size constraints - minimum width accommodates toolbar UI minSize = NSSize(width: 300, height: 200) @@ -198,7 +196,23 @@ final class PreviewWindow: NSPanel { } private func handleSpecialKeys(_ event: NSEvent) -> Bool { - if handleEscape(event) { return true } + let isEditingText = viewModel.isWaitingForTextInput || (firstResponder is NSTextView) + + if handleEscape(event, isEditingText: isEditingText) { return true } + if isEditingText { + // 多行 TextEditor 中 Enter 保留换行语义;⌘Enter 显式提交, + // 点击画布空白处同样提交、Esc 取消。 + if event.modifierFlags.contains(.command), + event.keyCode == 36 || event.keyCode == 76 { + Task { @MainActor in + viewModel.commitTextInput() + } + return true + } + // Let standard text editing keys (typing letters, backspace, Cmd+A/C/V/X/Z) go to the text field + return false + } + if handleDelete(event) { return true } if handleReturn(event) { return true } if handleCommandShortcuts(event) { return true } @@ -206,10 +220,12 @@ final class PreviewWindow: NSPanel { return false } - private func handleEscape(_ event: NSEvent) -> Bool { + private func handleEscape(_ event: NSEvent, isEditingText: Bool) -> Bool { guard event.keyCode == 53 else { return false } Task { @MainActor in - if viewModel.selectedAnnotationIndex != nil { + if isEditingText || viewModel.isWaitingForTextInput { + viewModel.cancelCurrentDrawing() + } else if viewModel.selectedAnnotationIndex != nil { viewModel.deselectAnnotation() } else if viewModel.selectedTool != nil { viewModel.selectTool(nil) @@ -269,7 +285,9 @@ final class PreviewWindow: NSPanel { } private func handleToolShortcuts(_ event: NSEvent) -> Bool { - guard let char = event.charactersIgnoringModifiers?.lowercased().first else { return false } + // ⌘ 组合保留给系统语义(⌘X 剪切、⌘C 复制等),不触发工具切换 + guard !event.modifierFlags.contains(.command), + let char = event.charactersIgnoringModifiers?.lowercased().first else { return false } if char == "c" && !event.modifierFlags.contains(.command) { Task { @MainActor in viewModel.toggleCropMode() } @@ -304,6 +322,12 @@ final class PreviewWindow: NSPanel { case "n": Task { @MainActor in toggleTool(.numberLabel) } return true + case "x": + Task { @MainActor in toggleTool(.privacyRedaction) } + return true + case "b": + Task { @MainActor in toggleTool(.calloutBubble) } + return true case "p": Task { @MainActor in viewModel.pinScreenshot() } return true @@ -396,21 +420,29 @@ final class PreviewWindowController: NSWindowController { /// - onSave: Callback when the screenshot is saved func showPreview( for screenshot: Screenshot, + autoTriggerOCR: Bool = false, + autoTriggerTranslate: Bool = false, onSave: @escaping (URL) -> Void = { _ in } ) { // Close any existing preview closePreview() // Create new preview window - previewWindow = PreviewWindow( + let win = PreviewWindow( screenshot: screenshot, onDismiss: { [weak self] in self?.closePreview() }, onSave: onSave ) - - previewWindow?.showPreview() + if autoTriggerOCR { + win.triggerOCR() + } + if autoTriggerTranslate { + win.triggerTranslation() + } + previewWindow = win + win.showPreview() } /// Closes the current preview window @@ -418,4 +450,18 @@ final class PreviewWindowController: NSWindowController { previewWindow?.closePreview() previewWindow = nil } + + func triggerOCR() { + previewWindow?.triggerOCR() + } +} + +extension PreviewWindow { + func triggerOCR() { + viewModel.performOCR() + } + + func triggerTranslation() { + viewModel.performTranslation() + } } diff --git a/TransFrame/Features/Settings/SettingsViewModel.swift b/TransFrame/Features/Settings/SettingsViewModel.swift index f90a498..4bbb668 100644 --- a/TransFrame/Features/Settings/SettingsViewModel.swift +++ b/TransFrame/Features/Settings/SettingsViewModel.swift @@ -1037,7 +1037,7 @@ final class SettingsViewModel { } request.httpBody = try JSONSerialization.data(withJSONObject: body) - let (_, response) = try await URLSession.shared.data(for: request) + let (data, response) = try await URLSession.shared.data(for: request) guard let httpResponse = response as? HTTPURLResponse else { throw VLMProviderError.invalidResponse("Invalid HTTP response") @@ -1045,6 +1045,9 @@ final class SettingsViewModel { switch httpResponse.statusCode { case 200: + // 与实际分析共用同一解析器:HTTP 200 但正文无法解析时如实报错, + // 避免"测试通过、实际识别失败"的错位(2026-10-23 用户反馈)。 + _ = try OpenAIVLMProvider.extractContentAndStatus(from: data) return (true, String(format: NSLocalizedString("settings.vlm.test.success", comment: ""), modelName)) case 401: throw VLMProviderError.authenticationFailed diff --git a/TransFrame/Features/TranslationFlow/BilingualOverlayView.swift b/TransFrame/Features/TranslationFlow/BilingualOverlayView.swift new file mode 100644 index 0000000..0c1f873 --- /dev/null +++ b/TransFrame/Features/TranslationFlow/BilingualOverlayView.swift @@ -0,0 +1,70 @@ +import SwiftUI +import AppKit + +/// 在截图画布上显示译文,原文由底下的原始截图保留。 +struct BilingualOverlayView: View { + let segments: [BilingualSegment] + + var body: some View { + Canvas { context, size in + for segment in segments { + guard let strip = BilingualOverlayLayout.translationStrip(for: segment, in: size) else { continue } + let text = Text(segment.translated) + .font(.system(size: strip.fontSize, weight: .medium)) + .foregroundStyle(.white) + context.fill( + Path(roundedRect: strip.frame, cornerRadius: 3), + with: .color(.black.opacity(0.78)) + ) + context.draw(context.resolve(text), in: strip.textFrame) + } + } + .allowsHitTesting(false) + .accessibilityHidden(true) + } +} + +enum BilingualOverlayLayout { + struct Anchor: Equatable { + let x: CGFloat + let y: CGFloat + let maxWidth: CGFloat + let fontSize: CGFloat + } + + struct Strip { + let frame: CGRect + let fontSize: CGFloat + + var textFrame: CGRect { frame.insetBy(dx: 4, dy: 2) } + } + + /// TextSegment 使用左上角原点;译文从原文下边缘开始,按画布宽度换行。 + static func translationAnchor(for boundingBox: CGRect, in displaySize: CGSize) -> Anchor { + let x = max(0, min(boundingBox.minX * displaySize.width, displaySize.width)) + let y = boundingBox.maxY * displaySize.height + 1 + let stripWidth = min(max(boundingBox.width * displaySize.width, 60), max(0, displaySize.width - x)) + let fontSize = min(max(boundingBox.height * displaySize.height * 0.85, 10), 16) + return Anchor(x: x, y: y, maxWidth: stripWidth, fontSize: fontSize) + } + + /// AppKit 与 SwiftUI 共用实际文字测量,避免换行后被条带高度裁掉。 + static func translationStrip(for segment: BilingualSegment, in displaySize: CGSize) -> Strip? { + let box = segment.boundingBox + guard displaySize.width > 0, displaySize.height > 0, + box.minX.isFinite, box.minY.isFinite, box.width.isFinite, box.height.isFinite, + box.width > 0, box.height > 0, + !segment.translated.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + let anchor = translationAnchor(for: box, in: displaySize) + guard anchor.maxWidth > 8, anchor.y < displaySize.height else { return nil } + let measured = (segment.translated as NSString).boundingRect( + with: CGSize(width: anchor.maxWidth - 8, height: CGFloat.greatestFiniteMagnitude), + options: [.usesLineFragmentOrigin, .usesFontLeading], + attributes: [.font: NSFont.systemFont(ofSize: anchor.fontSize, weight: .medium)] + ) + return Strip( + frame: CGRect(x: anchor.x, y: anchor.y, width: anchor.maxWidth, height: ceil(measured.height) + 4), + fontSize: anchor.fontSize + ) + } +} diff --git a/TransFrame/Models/Annotation.swift b/TransFrame/Models/Annotation.swift index 6c6c3f4..6a94ed5 100644 --- a/TransFrame/Models/Annotation.swift +++ b/TransFrame/Models/Annotation.swift @@ -1,5 +1,6 @@ import Foundation import CoreGraphics +import AppKit /// A drawing element placed on a screenshot. /// Supports rectangle, freehand, arrow, text, ellipse, line, mosaic, highlight, and numberLabel annotation types. @@ -87,14 +88,28 @@ struct RectangleAnnotation: Identifiable, Equatable, Sendable { var style: StrokeStyle /// Whether the rectangle is filled (solid) or hollow (outline only) - /// When filled, the rectangle uses the stroke color as fill to hide underlying content var isFilled: Bool - init(id: UUID = UUID(), rect: CGRect, style: StrokeStyle = .default, isFilled: Bool = false) { + /// Background fill color used when filled; nil falls back to the stroke color (legacy documents) + var fillColor: CodableColor? + + init( + id: UUID = UUID(), + rect: CGRect, + style: StrokeStyle = .default, + isFilled: Bool = false, + fillColor: CodableColor? = nil + ) { self.id = id self.rect = rect self.style = style self.isFilled = isFilled + self.fillColor = fillColor + } + + /// The color actually painted inside a filled rectangle + var effectiveFillColor: CodableColor { + fillColor ?? style.color } } @@ -145,6 +160,7 @@ struct FreehandAnnotation: Identifiable, Equatable, Sendable { // MARK: - Arrow Annotation /// An arrow annotation with start point, end point (arrowhead), and stroke style. +/// Supports straight arrow or curved arrow via quadratic bezier control point. struct ArrowAnnotation: Identifiable, Equatable, Sendable { /// Unique identifier let id: UUID @@ -155,14 +171,18 @@ struct ArrowAnnotation: Identifiable, Equatable, Sendable { /// End point of the arrow (head) in image coordinates var endPoint: CGPoint + /// Optional curvature control point for curved arrows + var controlPoint: CGPoint? + /// Stroke color and line width var style: StrokeStyle - init(id: UUID = UUID(), startPoint: CGPoint, endPoint: CGPoint, style: StrokeStyle = .default) { + init(id: UUID = UUID(), startPoint: CGPoint, endPoint: CGPoint, style: StrokeStyle = .default, controlPoint: CGPoint? = nil) { self.id = id self.startPoint = startPoint self.endPoint = endPoint self.style = style + self.controlPoint = controlPoint } /// Whether this annotation has meaningful length @@ -175,10 +195,17 @@ struct ArrowAnnotation: Identifiable, Equatable, Sendable { /// The bounding rectangle of the arrow var bounds: CGRect { - let minX = min(startPoint.x, endPoint.x) - let minY = min(startPoint.y, endPoint.y) - let maxX = max(startPoint.x, endPoint.x) - let maxY = max(startPoint.y, endPoint.y) + var minX = min(startPoint.x, endPoint.x) + var minY = min(startPoint.y, endPoint.y) + var maxX = max(startPoint.x, endPoint.x) + var maxY = max(startPoint.y, endPoint.y) + + if let cp = controlPoint { + minX = min(minX, cp.x) + minY = min(minY, cp.y) + maxX = max(maxX, cp.x) + maxY = max(maxY, cp.y) + } // Add padding for the arrowhead let padding: CGFloat = style.lineWidth * 3 @@ -198,7 +225,7 @@ struct TextAnnotation: Identifiable, Equatable, Sendable { /// Unique identifier let id: UUID - /// Anchor point in image coordinates + /// Anchor point in image coordinates (top-leading corner of the text block) var position: CGPoint /// User-entered text content @@ -207,11 +234,21 @@ struct TextAnnotation: Identifiable, Equatable, Sendable { /// Font, size, and color var style: TextStyle - init(id: UUID = UUID(), position: CGPoint, content: String, style: TextStyle = .default) { + /// Maximum line width for automatic word wrapping; nil = single line without wrapping + var wrapWidth: CGFloat? + + init( + id: UUID = UUID(), + position: CGPoint, + content: String, + style: TextStyle = .default, + wrapWidth: CGFloat? = nil + ) { self.id = id self.position = position self.content = content self.style = style + self.wrapWidth = wrapWidth } /// Whether this annotation has non-empty content @@ -219,19 +256,49 @@ struct TextAnnotation: Identifiable, Equatable, Sendable { !content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } - /// Estimated bounds of the text annotation based on font size and content + /// Caches text measurements keyed by (fontName, fontSize, wrapWidth, content). + /// `bounds` runs on hit-test and handle-drag frames; re-typesetting the + /// same text every frame stutters once many text annotations exist. + /// NSCache is thread-safe, evicts under pressure, and cached sizes stay + /// correct because the key covers every measurement input. + nonisolated(unsafe) private static let measuredBoundsCache = NSCache() + + /// Bounds of the text annotation measured with the actual font so multi-line + /// content is never truncated. var bounds: CGRect { - // Estimate text size based on font size and character count - // Average character width is approximately 0.6 × font size for most fonts - let averageCharWidth = style.fontSize * 0.6 - let estimatedWidth = max(CGFloat(content.count) * averageCharWidth, style.fontSize * 2) - // Height is approximately 1.2 × font size (line height) - let estimatedHeight = style.fontSize * 1.3 + guard !content.isEmpty else { + return CGRect(origin: position, size: CGSize(width: style.fontSize * 2, height: style.fontSize * 1.3)) + } - return CGRect( - origin: position, - size: CGSize(width: estimatedWidth, height: estimatedHeight) + let constraintWidth = wrapWidth ?? .greatestFiniteMagnitude + let wrapKey = wrapWidth.map { String(format: "%.1f", $0) } ?? "inf" + let cacheKey = "\(style.fontName)|\(String(format: "%.2f", style.fontSize))|\(wrapKey)|\(content)" as NSString + if let cached = Self.measuredBoundsCache.object(forKey: cacheKey) { + return CGRect(origin: position, size: cached.rectValue.size) + } + + let font = NSFont(name: style.fontName, size: style.fontSize) ?? NSFont.systemFont(ofSize: style.fontSize) + let paragraph = NSMutableParagraphStyle() + paragraph.lineBreakMode = .byWordWrapping + let attributes: [NSAttributedString.Key: Any] = [ + .font: font, + .paragraphStyle: paragraph + ] + + let measured = (content as NSString).boundingRect( + with: CGSize(width: constraintWidth, height: .greatestFiniteMagnitude), + options: [.usesLineFragmentOrigin, .usesFontLeading], + attributes: attributes, + context: nil + ) + + // Add a small horizontal tolerance so the trailing glyph is fully covered + let size = CGSize(width: ceil(measured.width) + 1, height: ceil(measured.height)) + Self.measuredBoundsCache.setObject( + NSValue(rect: CGRect(origin: .zero, size: size)), + forKey: cacheKey ) + return CGRect(origin: position, size: size) } } @@ -251,17 +318,32 @@ struct EllipseAnnotation: Identifiable, Equatable, Sendable { /// Whether the ellipse is filled (solid) or hollow (outline only) var isFilled: Bool - init(id: UUID = UUID(), rect: CGRect, style: StrokeStyle = .default, isFilled: Bool = false) { + /// Background fill color used when filled; nil falls back to the stroke color (legacy documents) + var fillColor: CodableColor? + + init( + id: UUID = UUID(), + rect: CGRect, + style: StrokeStyle = .default, + isFilled: Bool = false, + fillColor: CodableColor? = nil + ) { self.id = id self.rect = rect self.style = style self.isFilled = isFilled + self.fillColor = fillColor } /// Whether this annotation has meaningful size var isValid: Bool { rect.width >= 5 && rect.height >= 5 } + + /// The color actually painted inside a filled ellipse + var effectiveFillColor: CodableColor { + fillColor ?? style.color + } } // MARK: - Line Annotation @@ -328,7 +410,7 @@ struct MosaicAnnotation: Identifiable, Equatable, Sendable { init(id: UUID = UUID(), rect: CGRect, blockSize: Int = 10) { self.id = id self.rect = rect - self.blockSize = max(4, min(32, blockSize)) + self.blockSize = max(4, min(128, blockSize)) } /// Whether this annotation has meaningful size @@ -385,16 +467,26 @@ struct NumberLabelAnnotation: Identifiable, Equatable, Sendable { /// Text color var color: CodableColor - init(id: UUID = UUID(), position: CGPoint, number: Int, size: CGFloat = 24, color: CodableColor = CodableColor(.red)) { + /// Optional lead point for step annotation pointing arrow/line + var leadPoint: CGPoint? + + init(id: UUID = UUID(), position: CGPoint, number: Int, size: CGFloat = 24, color: CodableColor = CodableColor(.red), leadPoint: CGPoint? = nil) { self.id = id self.position = position self.number = max(1, number) - self.size = max(16, min(64, size)) + self.size = Self.clampedSize(size) self.color = color + self.leadPoint = leadPoint } - /// The bounding rectangle centered at position - var bounds: CGRect { + /// Uniform diameter clamp shared by init, scaled(by:), and handle drags + /// so a resize never jumps a 12–16pt label up to 16pt. + static func clampedSize(_ size: CGFloat) -> CGFloat { + max(12, min(256, size)) + } + + /// The circle bounding rectangle centered at position (excluding leadPoint) + var circleRect: CGRect { CGRect( x: position.x - size / 2, y: position.y - size / 2, @@ -402,6 +494,15 @@ struct NumberLabelAnnotation: Identifiable, Equatable, Sendable { height: size ) } + + /// The bounding rectangle centered at position, expanded to cover leadPoint if set + var bounds: CGRect { + var rect = circleRect + if let lp = leadPoint { + rect = rect.union(CGRect(x: lp.x - 4, y: lp.y - 4, width: 8, height: 8)) + } + return rect + } } // MARK: - CGPoint Sendable Conformance @@ -499,6 +600,19 @@ struct CalloutBubbleAnnotation: Identifiable, Equatable, Sendable { } } +// MARK: - Geometry Constraints + +/// Pure geometry helpers shared by drawing tools and the in-place overlay. +enum GeometryConstraint { + /// 1:1 square bounding box from a start point toward a drag point (⌘ constraint). + static func squareRect(from start: CGPoint, to end: CGPoint) -> CGRect { + let side = max(abs(end.x - start.x), abs(end.y - start.y)) + let originX = end.x >= start.x ? start.x : start.x - side + let originY = end.y >= start.y ? start.y : start.y - side + return CGRect(x: originX, y: originY, width: side, height: side) + } +} + // MARK: - Precise Geometric Hit Testing extension Annotation { @@ -522,9 +636,28 @@ extension Annotation { return d <= (l.style.lineWidth / 2 + tolerance) case .arrow(let a): - let d = distanceToSegment(p: point, a: a.startPoint, b: a.endPoint) - if d <= (a.style.lineWidth / 2 + tolerance) { return true } - return a.bounds.insetBy(dx: -tolerance, dy: -tolerance).contains(point) + let threshold = a.style.lineWidth / 2 + tolerance + if let cp = a.controlPoint { + let steps = 16 + var prev = a.startPoint + for i in 1...steps { + let t = CGFloat(i) / CGFloat(steps) + let oneMinusT = 1.0 - t + let cur = CGPoint( + x: oneMinusT * oneMinusT * a.startPoint.x + 2 * oneMinusT * t * cp.x + t * t * a.endPoint.x, + y: oneMinusT * oneMinusT * a.startPoint.y + 2 * oneMinusT * t * cp.y + t * t * a.endPoint.y + ) + if distanceToSegment(p: point, a: prev, b: cur) <= threshold { + return true + } + prev = cur + } + } else { + let d = distanceToSegment(p: point, a: a.startPoint, b: a.endPoint) + if d <= threshold { return true } + } + let headRadius = max(a.style.lineWidth * 3.5, 12) + tolerance + return hypot(point.x - a.endPoint.x, point.y - a.endPoint.y) <= headRadius case .freehand(let f): guard f.points.count >= 2 else { return false } @@ -546,13 +679,26 @@ extension Annotation { return h.rect.insetBy(dx: -tolerance, dy: -tolerance).contains(point) case .numberLabel(let n): - return n.bounds.insetBy(dx: -tolerance, dy: -tolerance).contains(point) + if hypot(point.x - n.position.x, point.y - n.position.y) <= (n.size / 2 + tolerance) { + return true + } + if let lp = n.leadPoint { + return distanceToSegment(p: point, a: n.position, b: lp) <= (2.0 + tolerance) + } + return false case .privacyRedaction(let pr): return pr.rect.insetBy(dx: -tolerance, dy: -tolerance).contains(point) case .calloutBubble(let cb): - return cb.bounds.insetBy(dx: -tolerance, dy: -tolerance).contains(point) + if cb.rect.insetBy(dx: -tolerance, dy: -tolerance).contains(point) { + return true + } + let attachPoint = CGPoint( + x: max(cb.rect.minX + 8, min(cb.rect.maxX - 8, cb.targetPoint.x)), + y: cb.targetPoint.y < cb.rect.minY ? cb.rect.minY : (cb.targetPoint.y > cb.rect.maxY ? cb.rect.maxY : cb.rect.midY) + ) + return distanceToSegment(p: point, a: attachPoint, b: cb.targetPoint) <= (4.0 + tolerance) } } @@ -595,4 +741,474 @@ extension Annotation { let innerRect = rect.insetBy(dx: effTol, dy: effTol) return isPointInsideEllipse(p, rect: outerRect) && !isPointInsideEllipse(p, rect: innerRect) } + + /// Returns a new annotation shifted by the specified delta. + func offsetBy(dx: CGFloat, dy: CGFloat) -> Annotation { + switch self { + case .rectangle(var r): + r.rect.origin.x += dx; r.rect.origin.y += dy + return .rectangle(r) + case .ellipse(var e): + e.rect.origin.x += dx; e.rect.origin.y += dy + return .ellipse(e) + case .line(var l): + l.startPoint.x += dx; l.startPoint.y += dy + l.endPoint.x += dx; l.endPoint.y += dy + return .line(l) + case .freehand(var f): + f.points = f.points.map { CGPoint(x: $0.x + dx, y: $0.y + dy) } + return .freehand(f) + case .arrow(var a): + a.startPoint.x += dx; a.startPoint.y += dy + a.endPoint.x += dx; a.endPoint.y += dy + if let cp = a.controlPoint { + a.controlPoint = CGPoint(x: cp.x + dx, y: cp.y + dy) + } + return .arrow(a) + case .highlight(var h): + h.rect.origin.x += dx; h.rect.origin.y += dy + return .highlight(h) + case .mosaic(var m): + m.rect.origin.x += dx; m.rect.origin.y += dy + return .mosaic(m) + case .text(var t): + t.position.x += dx; t.position.y += dy + return .text(t) + case .numberLabel(var n): + n.position.x += dx; n.position.y += dy + if let lp = n.leadPoint { + n.leadPoint = CGPoint(x: lp.x + dx, y: lp.y + dy) + } + return .numberLabel(n) + case .privacyRedaction(var pr): + pr.rect.origin.x += dx; pr.rect.origin.y += dy + return .privacyRedaction(pr) + case .calloutBubble(var cb): + cb.rect.origin.x += dx; cb.rect.origin.y += dy + cb.targetPoint.x += dx; cb.targetPoint.y += dy + return .calloutBubble(cb) + } + } + + /// Returns a new annotation with updated color if applicable. + func updatingColor(_ color: CodableColor) -> Annotation { + switch self { + case .rectangle(var r): + r.style.color = color; return .rectangle(r) + case .ellipse(var e): + e.style.color = color; return .ellipse(e) + case .line(var l): + l.style.color = color; return .line(l) + case .freehand(var f): + f.style.color = color; return .freehand(f) + case .arrow(var a): + a.style.color = color; return .arrow(a) + case .highlight(var h): + h.color = color; return .highlight(h) + case .mosaic: + return self + case .text(var t): + t.style.color = color; return .text(t) + case .numberLabel(var n): + n.color = color; return .numberLabel(n) + case .privacyRedaction(var pr): + pr.fillColor = color; return .privacyRedaction(pr) + case .calloutBubble(var cb): + cb.bubbleColor = color; return .calloutBubble(cb) + } + } + + /// Returns a new annotation with an updated background fill color if applicable. + func updatingFillColor(_ color: CodableColor) -> Annotation { + switch self { + case .rectangle(var r): + r.fillColor = color; return .rectangle(r) + case .ellipse(var e): + e.fillColor = color; return .ellipse(e) + default: + return self + } + } + + /// Returns a new annotation with updated line width if applicable. + func updatingLineWidth(_ width: CGFloat) -> Annotation { + switch self { + case .rectangle(var r): + r.style.lineWidth = width; return .rectangle(r) + case .ellipse(var e): + e.style.lineWidth = width; return .ellipse(e) + case .line(var l): + l.style.lineWidth = width; return .line(l) + case .freehand(var f): + f.style.lineWidth = width; return .freehand(f) + case .arrow(var a): + a.style.lineWidth = width; return .arrow(a) + default: + return self + } + } + + /// Scales all geometric coordinates and visual dimensions of this annotation by the given factor (e.g. Retina scaleFactor). + func scaled(by factor: CGFloat) -> Annotation { + guard factor > 0 && abs(factor - 1.0) > 0.001 else { return self } + let scaleRect: (CGRect) -> CGRect = { r in + CGRect( + x: r.origin.x * factor, + y: r.origin.y * factor, + width: r.size.width * factor, + height: r.size.height * factor + ) + } + let scalePoint: (CGPoint) -> CGPoint = { p in + CGPoint(x: p.x * factor, y: p.y * factor) + } + + switch self { + case .rectangle(var r): + r.rect = scaleRect(r.rect) + r.style.lineWidth *= factor + return .rectangle(r) + case .ellipse(var e): + e.rect = scaleRect(e.rect) + e.style.lineWidth *= factor + return .ellipse(e) + case .line(var l): + l.startPoint = scalePoint(l.startPoint) + l.endPoint = scalePoint(l.endPoint) + l.style.lineWidth *= factor + return .line(l) + case .freehand(var f): + f.points = f.points.map(scalePoint) + f.style.lineWidth *= factor + return .freehand(f) + case .arrow(var a): + a.startPoint = scalePoint(a.startPoint) + a.endPoint = scalePoint(a.endPoint) + if let cp = a.controlPoint { + a.controlPoint = scalePoint(cp) + } + a.style.lineWidth *= factor + return .arrow(a) + case .highlight(var h): + h.rect = scaleRect(h.rect) + return .highlight(h) + case .mosaic(var m): + m.rect = scaleRect(m.rect) + m.blockSize = max(4, min(128, Int(round(CGFloat(m.blockSize) * factor)))) + return .mosaic(m) + case .text(var t): + t.position = scalePoint(t.position) + t.style.fontSize *= factor + if let wrap = t.wrapWidth { + t.wrapWidth = wrap * factor + } + return .text(t) + case .numberLabel(var n): + n.position = scalePoint(n.position) + n.size = NumberLabelAnnotation.clampedSize(n.size * factor) + if let lp = n.leadPoint { + n.leadPoint = scalePoint(lp) + } + return .numberLabel(n) + case .privacyRedaction(var pr): + pr.rect = scaleRect(pr.rect) + return .privacyRedaction(pr) + case .calloutBubble(var cb): + cb.rect = scaleRect(cb.rect) + cb.targetPoint = scalePoint(cb.targetPoint) + cb.style.fontSize *= factor + return .calloutBubble(cb) + } + } +} + +// MARK: - Interactive Control Handles for Resizing & Point Manipulation + +enum AnnotationHandle: Equatable, Sendable { + case topLeft + case topRight + case bottomLeft + case bottomRight + case startPoint + case endPoint + case controlPoint + case leadPoint + case targetPoint + case leftEdge + case rightEdge + case topEdge + case bottomEdge + + var isSpecialPoint: Bool { + switch self { + case .controlPoint, .leadPoint, .targetPoint: + return true + default: + return false + } + } + + var cursor: NSCursor { + switch self { + case .topLeft, .bottomRight: + return NSCursor.diagonalResizeCursor(northWestSouthEast: true) + case .topRight, .bottomLeft: + return NSCursor.diagonalResizeCursor(northWestSouthEast: false) + case .leftEdge, .rightEdge: + return .resizeLeftRight + case .topEdge, .bottomEdge: + return .resizeUpDown + case .startPoint, .endPoint, .controlPoint, .leadPoint, .targetPoint: + return .crosshair + } + } +} + +/// Extra constraints applied while dragging an annotation control handle. +struct HandleDragOptions: Sendable { + /// Hold ⌘ while resizing to lock the current width/height ratio + var preserveAspectRatio: Bool = false + + static let standard = HandleDragOptions() +} + +extension Annotation { + /// Returns all interactive handles for this annotation in its coordinate space. + func controlHandles(padding: CGFloat = 0) -> [(handle: AnnotationHandle, point: CGPoint)] { + let cornerHandles: (CGRect) -> [(AnnotationHandle, CGPoint)] = { rect in + let r = rect.insetBy(dx: -padding, dy: -padding) + return [ + (.topLeft, CGPoint(x: r.minX, y: r.minY)), + (.topRight, CGPoint(x: r.maxX, y: r.minY)), + (.bottomLeft, CGPoint(x: r.minX, y: r.maxY)), + (.bottomRight, CGPoint(x: r.maxX, y: r.maxY)) + ] + } + + switch self { + case .rectangle(let r): + return cornerHandles(r.rect) + case .ellipse(let e): + return cornerHandles(e.rect) + case .mosaic(let m): + return cornerHandles(m.rect) + case .highlight(let h): + return cornerHandles(h.rect) + case .privacyRedaction(let pr): + return cornerHandles(pr.rect) + case .freehand(let f): + return cornerHandles(f.bounds) + case .line(let l): + return [ + (.startPoint, l.startPoint), + (.endPoint, l.endPoint) + ] + case .arrow(let a): + let controlPos = a.controlPoint ?? CGPoint( + x: (a.startPoint.x + a.endPoint.x) / 2, + y: (a.startPoint.y + a.endPoint.y) / 2 + ) + return [ + (.startPoint, a.startPoint), + (.controlPoint, controlPos), + (.endPoint, a.endPoint) + ] + case .numberLabel(let n): + let leadPos = n.leadPoint ?? CGPoint(x: n.position.x + n.size / 2 + 8, y: n.position.y) + let resizePos = CGPoint( + x: n.position.x + (n.size / 2 + padding) * 0.7071, + y: n.position.y + (n.size / 2 + padding) * 0.7071 + ) + return [ + (.leadPoint, leadPos), + (.bottomRight, resizePos) + ] + case .calloutBubble(let cb): + return cornerHandles(cb.rect) + [(.targetPoint, cb.targetPoint)] + case .text(let t): + let b = t.bounds.insetBy(dx: -padding, dy: -padding) + return [ + (.topLeft, CGPoint(x: b.minX, y: b.minY)), + (.topRight, CGPoint(x: b.maxX, y: b.minY)), + (.bottomLeft, CGPoint(x: b.minX, y: b.maxY)), + (.bottomRight, CGPoint(x: b.maxX, y: b.maxY)), + (.topEdge, CGPoint(x: b.midX, y: b.minY)), + (.bottomEdge, CGPoint(x: b.midX, y: b.maxY)), + (.leftEdge, CGPoint(x: b.minX, y: b.midY)), + (.rightEdge, CGPoint(x: b.maxX, y: b.midY)) + ] + } + } + + /// Tests if a point hits any of this annotation's control handles. + func hitTestHandle(at point: CGPoint, padding: CGFloat = 0, tolerance: CGFloat = 10.0) -> AnnotationHandle? { + for (handle, handlePoint) in controlHandles(padding: padding) { + if hypot(point.x - handlePoint.x, point.y - handlePoint.y) <= tolerance { + return handle + } + } + return nil + } + + /// Returns a modified annotation after dragging the specified handle to a new point. + /// - Parameter options: Extra constraints, e.g. ⌘-held proportional resizing. + func applyingHandleDrag( + _ handle: AnnotationHandle, + to point: CGPoint, + options: HandleDragOptions = .standard + ) -> Annotation { + let resizedRect: (CGRect, CGFloat, CGFloat) -> CGRect = { rect, minW, minH in + let anchor: CGPoint + switch handle { + case .topLeft: + anchor = CGPoint(x: rect.maxX, y: rect.maxY) + case .topRight: + anchor = CGPoint(x: rect.minX, y: rect.maxY) + case .bottomLeft: + anchor = CGPoint(x: rect.maxX, y: rect.minY) + case .bottomRight: + anchor = CGPoint(x: rect.minX, y: rect.minY) + default: + return rect + } + var width = max(minW, abs(point.x - anchor.x)) + var height = max(minH, abs(point.y - anchor.y)) + + // ⌘ held: lock the pre-drag width/height ratio + if options.preserveAspectRatio, rect.width > 0.001, rect.height > 0.001 { + let aspect = rect.width / rect.height + if width / max(height, 0.001) > aspect { + height = width / aspect + } else { + width = height * aspect + } + } + + let dirX: CGFloat = point.x >= anchor.x ? 1 : -1 + let dirY: CGFloat = point.y >= anchor.y ? 1 : -1 + let minX = dirX >= 0 ? anchor.x : anchor.x - width + let minY = dirY >= 0 ? anchor.y : anchor.y - height + return CGRect(x: minX, y: minY, width: width, height: height) + } + + switch self { + case .rectangle(var r): + r.rect = resizedRect(r.rect, 6, 6) + return .rectangle(r) + case .ellipse(var e): + e.rect = resizedRect(e.rect, 6, 6) + return .ellipse(e) + case .mosaic(var m): + m.rect = resizedRect(m.rect, 10, 10) + return .mosaic(m) + case .highlight(var h): + h.rect = resizedRect(h.rect, 6, 6) + return .highlight(h) + case .privacyRedaction(var pr): + pr.rect = resizedRect(pr.rect, 6, 6) + return .privacyRedaction(pr) + case .freehand(var f): + let oldBounds = f.bounds + guard oldBounds.width >= 2, oldBounds.height >= 2 else { return self } + let newBounds = resizedRect(oldBounds, 8, 8) + f.points = f.points.map { p in + let nx = (p.x - oldBounds.minX) / oldBounds.width + let ny = (p.y - oldBounds.minY) / oldBounds.height + return CGPoint( + x: newBounds.minX + nx * newBounds.width, + y: newBounds.minY + ny * newBounds.height + ) + } + return .freehand(f) + case .line(var l): + if handle == .startPoint { + l.startPoint = point + } else if handle == .endPoint { + l.endPoint = point + } + return .line(l) + case .arrow(var a): + switch handle { + case .startPoint: + a.startPoint = point + case .endPoint: + a.endPoint = point + case .controlPoint: + a.controlPoint = point + default: + break + } + return .arrow(a) + case .numberLabel(var n): + switch handle { + case .leadPoint: + let dist = hypot(point.x - n.position.x, point.y - n.position.y) + n.leadPoint = dist > (n.size / 2 + 4) ? point : nil + case .bottomRight: + let dist = hypot(point.x - n.position.x, point.y - n.position.y) + n.size = NumberLabelAnnotation.clampedSize(dist * 2) + default: + break + } + return .numberLabel(n) + case .calloutBubble(var cb): + if handle == .targetPoint { + cb.targetPoint = point + } else { + cb.rect = resizedRect(cb.rect, 30, 22) + } + return .calloutBubble(cb) + case .text(var t): + let oldBounds = t.bounds + switch handle { + case .topLeft, .topRight, .bottomLeft, .bottomRight: + // Corner handle: scale font size (and wrap width) proportionally + let anchor: CGPoint + switch handle { + case .topLeft: + anchor = CGPoint(x: oldBounds.maxX, y: oldBounds.maxY) + case .topRight: + anchor = CGPoint(x: oldBounds.minX, y: oldBounds.maxY) + case .bottomLeft: + anchor = CGPoint(x: oldBounds.maxX, y: oldBounds.minY) + default: + anchor = CGPoint(x: oldBounds.minX, y: oldBounds.minY) + } + let oldDiagonal = hypot(oldBounds.width, oldBounds.height) + let newDiagonal = hypot(abs(point.x - anchor.x), abs(point.y - anchor.y)) + let factor = max(0.05, newDiagonal / max(oldDiagonal, 1)) + t.style.fontSize = max(8, min(144, t.style.fontSize * factor)) + if let wrap = t.wrapWidth { + t.wrapWidth = max(t.style.fontSize, wrap * factor) + } + // Keep the anchor corner fixed after re-measuring + let newBounds = t.bounds + t.position.x += anchor.x - (handle == .topLeft || handle == .bottomLeft ? newBounds.maxX : newBounds.minX) + t.position.y += anchor.y - (handle == .topLeft || handle == .topRight ? newBounds.maxY : newBounds.minY) + case .leftEdge: + // Left edge: change the wrap width, keeping the right edge fixed + let right = oldBounds.maxX + t.wrapWidth = max(t.style.fontSize, right - point.x) + t.position.x = right - t.bounds.width + case .rightEdge: + // Right edge: change the wrap width, keeping the left edge fixed + t.wrapWidth = max(t.style.fontSize, point.x - oldBounds.minX) + case .topEdge, .bottomEdge: + // Top/bottom edge: scale the font size from the vertical drag distance + let anchorY = handle == .topEdge ? oldBounds.maxY : oldBounds.minY + let newHeight = max(10, abs(point.y - anchorY)) + let factor = max(0.1, newHeight / max(oldBounds.height, 1)) + t.style.fontSize = max(8, min(144, t.style.fontSize * factor)) + if let wrap = t.wrapWidth { + t.wrapWidth = max(t.style.fontSize, wrap * factor) + } + // Keep the anchor edge fixed after re-measuring + let newBounds = t.bounds + t.position.y += anchorY - (handle == .topEdge ? newBounds.maxY : newBounds.minY) + default: + break + } + return .text(t) + } + } } diff --git a/TransFrame/Models/AppSettings.swift b/TransFrame/Models/AppSettings.swift index 48b0d0d..937aa76 100644 --- a/TransFrame/Models/AppSettings.swift +++ b/TransFrame/Models/AppSettings.swift @@ -27,63 +27,6 @@ enum PaddleOCRMode: String, Codable, CaseIterable, Sendable { } } -/// GLM OCR backend mode selection -enum GLMOCRMode: String, Codable, CaseIterable, Sendable { - case cloud - case local - - var localizedName: String { - switch self { - case .cloud: - return NSLocalizedString("settings.glmocr.mode.cloud", comment: "Cloud mode") - case .local: - return NSLocalizedString("settings.glmocr.mode.local", comment: "Local mode") - } - } - - var providerDescription: String { - switch self { - case .cloud: - return NSLocalizedString( - "vlm.provider.glmocr.description", - comment: "Zhipu GLM-OCR layout parsing API" - ) - case .local: - return NSLocalizedString( - "vlm.provider.glmocr.local.description", - comment: "Local MLX-VLM server for GLM-OCR" - ) - } - } - - var defaultBaseURL: String { - switch self { - case .cloud: - return "https://open.bigmodel.cn/api/paas/v4" - case .local: - return "http://127.0.0.1:18081" - } - } - - var defaultModelName: String { - switch self { - case .cloud: - return "glm-ocr" - case .local: - return "mlx-community/GLM-OCR-bf16" - } - } - - var requiresAPIKey: Bool { - switch self { - case .cloud: - return true - case .local: - return false - } - } -} - /// User preferences persisted across sessions via UserDefaults. /// All properties automatically sync to UserDefaults with the `TransFrame.` prefix. @MainActor @@ -112,6 +55,7 @@ final class AppSettings { static let textSize = prefix + "textSize" static let rectangleFilled = prefix + "rectangleFilled" static let ellipseFilled = prefix + "ellipseFilled" + static let shapeFillColor = prefix + "shapeFillColor" static let mosaicBlockSize = prefix + "mosaicBlockSize" static let translationTargetLanguage = prefix + "translationTargetLanguage" static let translationSourceLanguage = prefix + "translationSourceLanguage" @@ -231,6 +175,11 @@ final class AppSettings { didSet { save(ellipseFilled, forKey: Keys.ellipseFilled) } } + /// Default background fill color for solid rectangles/ellipses + var shapeFillColor: CodableColor { + didSet { saveColor(shapeFillColor, forKey: Keys.shapeFillColor) } + } + /// Default mosaic block size (pixelation level) var mosaicBlockSize: CGFloat { didSet { save(Double(mosaicBlockSize), forKey: Keys.mosaicBlockSize) } @@ -532,8 +481,10 @@ final class AppSettings { let defaults = UserDefaults.standard // Load save location from bookmark first, then path, or use Desktop + let isRunningTests = ProcessInfo.processInfo.environment["XCTestConfigurationFilePath"] != nil let loadedLocation: URL - if let bookmarkData = defaults.data(forKey: "SaveLocationBookmark"), + if !isRunningTests, + let bookmarkData = defaults.data(forKey: "SaveLocationBookmark"), let url = Self.resolveBookmark(bookmarkData) { loadedLocation = url } else if let path = defaults.string(forKey: Keys.saveLocation) { @@ -576,6 +527,7 @@ final class AppSettings { textSize = CGFloat(defaults.object(forKey: Keys.textSize) as? Double ?? 14.0) rectangleFilled = defaults.object(forKey: Keys.rectangleFilled) as? Bool ?? false ellipseFilled = defaults.object(forKey: Keys.ellipseFilled) as? Bool ?? false + shapeFillColor = Self.loadColor(forKey: Keys.shapeFillColor) ?? .yellow mosaicBlockSize = CGFloat(defaults.object(forKey: Keys.mosaicBlockSize) as? Double ?? 10.0) // Load translation settings @@ -694,6 +646,7 @@ final class AppSettings { textSize = 14.0 rectangleFilled = false ellipseFilled = false + shapeFillColor = .yellow mosaicBlockSize = 10.0 translationTargetLanguage = nil translationSourceLanguage = .auto diff --git a/TransFrame/Models/TranslationHistory.swift b/TransFrame/Models/TranslationHistory.swift index f97ff6b..829c24b 100644 --- a/TransFrame/Models/TranslationHistory.swift +++ b/TransFrame/Models/TranslationHistory.swift @@ -1,7 +1,33 @@ import Foundation -/// A single translation history entry. -/// Contains the source text, translated text, screenshot thumbnail, and metadata. +/// Supported history entry category types +public enum HistoryCategory: String, Codable, Sendable, CaseIterable { + case translation = "translation" + case screenshot = "screenshot" + case color = "color" + case upload = "upload" + + public var displayName: String { + switch self { + case .translation: return "Translation" + case .screenshot: return "Screenshot" + case .color: return "Color" + case .upload: return "Upload" + } + } + + public var systemImage: String { + switch self { + case .translation: return "character.book.closed" + case .screenshot: return "camera.viewfinder" + case .color: return "eyedropper" + case .upload: return "icloud.and.arrow.up" + } + } +} + +/// A single unified history entry. +/// Contains the source text, translated text, screenshot thumbnail, category, favorite state, and metadata. struct TranslationHistory: Identifiable, Codable, Sendable { // MARK: - Types @@ -14,6 +40,8 @@ struct TranslationHistory: Identifiable, Codable, Sendable { case targetLanguage case timestamp case thumbnailData + case category + case isFavorite } // MARK: - Properties @@ -21,24 +49,30 @@ struct TranslationHistory: Identifiable, Codable, Sendable { /// Unique identifier let id: UUID - /// Original source text + /// Original source text or primary description let sourceText: String - /// Translated text + /// Translated text, hex code, or public URL let translatedText: String - /// Source language name + /// Source language name or provider name let sourceLanguage: String /// Target language name let targetLanguage: String - /// When the translation was performed + /// When the entry was performed let timestamp: Date /// JPEG thumbnail data (max 10KB, 128px on longest edge) let thumbnailData: Data? + /// Category of history entry + var category: HistoryCategory + + /// Starred favorite state + var isFavorite: Bool + // MARK: - Initialization init( @@ -48,7 +82,9 @@ struct TranslationHistory: Identifiable, Codable, Sendable { sourceLanguage: String, targetLanguage: String, timestamp: Date = Date(), - thumbnailData: Data? = nil + thumbnailData: Data? = nil, + category: HistoryCategory = .translation, + isFavorite: Bool = false ) { self.id = id self.sourceText = sourceText @@ -57,6 +93,21 @@ struct TranslationHistory: Identifiable, Codable, Sendable { self.targetLanguage = targetLanguage self.timestamp = timestamp self.thumbnailData = thumbnailData + self.category = category + self.isFavorite = isFavorite + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.id = try container.decode(UUID.self, forKey: .id) + self.sourceText = try container.decode(String.self, forKey: .sourceText) + self.translatedText = try container.decode(String.self, forKey: .translatedText) + self.sourceLanguage = try container.decode(String.self, forKey: .sourceLanguage) + self.targetLanguage = try container.decode(String.self, forKey: .targetLanguage) + self.timestamp = try container.decode(Date.self, forKey: .timestamp) + self.thumbnailData = try container.decodeIfPresent(Data.self, forKey: .thumbnailData) + self.category = try container.decodeIfPresent(HistoryCategory.self, forKey: .category) ?? .translation + self.isFavorite = try container.decodeIfPresent(Bool.self, forKey: .isFavorite) ?? false } // MARK: - Computed Properties diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128.png index c64238b..8b410ad 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png index f758d95..bcfbb0c 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_128x128@2x.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16.png index a7043de..393341e 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png index 6211978..3169955 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_16x16@2x.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256.png index f758d95..bcfbb0c 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png index c9a4892..846d853 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_256x256@2x.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32.png index 6211978..3169955 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png index 868c6e2..7a34b36 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_32x32@2x.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512.png index c9a4892..846d853 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512.png differ diff --git a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png index 0f8d36c..7b5c857 100644 Binary files a/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png and b/TransFrame/Resources/Assets.xcassets/AppIcon.appiconset/icon_512x512@2x.png differ diff --git a/TransFrame/Resources/de.lproj/Localizable.strings b/TransFrame/Resources/de.lproj/Localizable.strings index 5a6d9e9..071af29 100644 --- a/TransFrame/Resources/de.lproj/Localizable.strings +++ b/TransFrame/Resources/de.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Texterkennung fehlgeschlagen."; "error.ocr.failed.recovery" = "Bitte versuchen Sie es erneut mit einem klareren Bild."; "error.ocr.no.text" = "Es wurde kein Text im Bild erkannt."; +"error.image.load.failed" = "Bild „%@“ konnte nicht geöffnet werden"; "error.ocr.no.text.recovery" = "Versuchen Sie, einen Bereich mit sichtbarem Text aufzunehmen."; "error.ocr.cancelled" = "Texterkennung wurde abgebrochen."; "error.ocr.server.unreachable" = "Keine Verbindung zum OCR-Server möglich."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Auswahl aufnehmen"; "menu.translation.mode" = "Übersetzungsmodus"; "menu.translation.history" = "Übersetzungsverlauf"; +"menu.pick.color" = "Farbe vom Bildschirm wählen"; +"menu.open.image" = "Bild zum Bearbeiten öffnen..."; "menu.settings" = "Einstellungen..."; "menu.about" = "Über TransFrame"; "menu.quit" = "TransFrame beenden"; diff --git a/TransFrame/Resources/en.lproj/Localizable.strings b/TransFrame/Resources/en.lproj/Localizable.strings index df65a2f..201bd5a 100644 --- a/TransFrame/Resources/en.lproj/Localizable.strings +++ b/TransFrame/Resources/en.lproj/Localizable.strings @@ -32,6 +32,8 @@ /* Export Errors */ "error.export.encoding.failed" = "Failed to encode the image."; "error.export.encoding.failed.recovery" = "Try a different format in Settings."; +"error.image.upload.failed" = "Image upload failed: %@"; +"error.image.upload.failed.recovery" = "Check the image host configuration and network, then retry."; "error.export.encoding.failed.detail" = "Failed to encode the image as %@."; /* Clipboard Errors */ @@ -46,8 +48,11 @@ /* OCR Errors */ "error.ocr.failed" = "Text recognition failed."; +"error.pipeline.ocrFailed" = "Text recognition (VLM) failed: %@"; +"error.pipeline.translateFailed" = "Translation failed: %@"; "error.ocr.failed.recovery" = "Please try again with a clearer image."; "error.ocr.no.text" = "No text was recognized in the image."; +"error.image.load.failed" = "Could not open image \"%@\""; "error.ocr.no.text.recovery" = "Try capturing an area with visible text."; "error.ocr.cancelled" = "Text recognition was cancelled."; "error.ocr.server.unreachable" = "Cannot connect to OCR server."; @@ -83,6 +88,8 @@ "menu.capture.fullscreen" = "Capture Full Screen"; "menu.capture.selection" = "Capture Selection"; "menu.translation.mode" = "Translation Mode"; +"menu.pick.color" = "Pick Screen Color"; +"menu.open.image" = "Open Image for Editing..."; "menu.translation.history" = "Translation History"; "menu.settings" = "Settings..."; "menu.about" = "About TransFrame"; @@ -136,6 +143,8 @@ "preview.tooltip.confirm" = "Copy to Clipboard and Close (Enter)"; "preview.tooltip.dismiss" = "Dismiss (Escape)"; "preview.tooltip.delete" = "Delete selected annotation"; +"preview.tooltip.editText" = "Edit Text"; +"preview.tooltip.beautify" = "Beautify (Background & Shadow)"; /* Accessibility Labels */ "preview.accessibility.save" = "Save screenshot"; @@ -164,6 +173,8 @@ "tool.highlight" = "Highlight"; "tool.mosaic" = "Mosaic"; "tool.numberLabel" = "Number Label"; +"tool.privacyRedaction" = "Privacy Redaction"; +"tool.calloutBubble" = "Callout Bubble"; /* ======================================== @@ -488,6 +499,8 @@ ======================================== */ "history.title" = "Translation History"; +"history.filter.all" = "All"; +"history.filter.starred" = "Starred"; "history.search.placeholder" = "Search history..."; "history.clear.all" = "Clear all history"; "history.empty.title" = "No Translation History"; @@ -706,6 +719,7 @@ ======================================== */ "bilingualResult.window.title" = "Bilingual Translation"; +"bilingualResult.pin" = "Pin"; "bilingualResult.loading" = "Translating..."; "bilingualResult.loading.analyzing" = "Analyzing image..."; "bilingualResult.loading.translating" = "Translating text..."; @@ -816,11 +830,6 @@ "about.acknowledgements.upstream" = "Based on"; "about.acknowledgements.author.format" = "by %@"; "about.close" = "Close"; -"settings.glmocr.mode" = "Mode"; -"settings.glmocr.mode.cloud" = "Cloud"; -"settings.glmocr.mode.local" = "Local"; -"settings.glmocr.local.apiKey.optional" = "API Key is optional for local MLX-VLM servers"; -"vlm.provider.glmocr.local.description" = "Local MLX-VLM server for GLM-OCR"; "engine.config.fetchModels" = "Fetch"; "engine.config.fetchModels.failed" = "Failed to fetch models"; @@ -830,3 +839,89 @@ "menu.pinned.show_all" = "Bring All Pins to Front"; "tool.privacyRedaction" = "Privacy Redaction"; "tool.calloutBubble" = "Callout Bubble"; + +/* Annotation color palette */ +"preview.color.custom" = "Custom Color"; +"preview.color.eyedropper" = "Pick Screen Color"; +"preview.fill.color" = "Fill Color"; + +/* Smart privacy mosaic */ +"smartMosaic.error.notConfigured" = "Please configure the VLM API key in Settings first (Smart Mosaic shares the VLM configuration)"; +"smartMosaic.detecting" = "Detecting privacy regions..."; +"smartMosaic.noneFound" = "No sensitive information detected"; +"smartMosaic.redacted" = "Masked %d privacy regions"; +"smartMosaic.error.failed" = "Privacy detection failed. Check your network and retry."; +"preview.tooltip.smartMosaic" = "Smart Mosaic"; + +/* Embedded results side panel */ +"preview.panel.ocrTab" = "Text Recognition"; +"preview.panel.translateTab" = "Translation"; +"preview.panel.rerun" = "Recognize Again"; +"preview.panel.copyAll" = "Copy All"; +"preview.panel.close" = "Close"; +"preview.panel.expandDrawer" = "Expand results panel"; +"preview.panel.recognizing" = "Recognizing..."; +"preview.panel.copyLine" = "Copy"; +"preview.panel.original" = "Original"; +"preview.panel.translation" = "Translation"; +"preview.panel.targetLanguage" = "Target Language"; +"preview.panel.noResults" = "No results yet. Click Text Recognition to start"; +"preview.panel.translating" = "Translating..."; +"preview.panel.translate" = "Translate"; +"mosaic.intensity.light" = "Light"; +"mosaic.intensity.medium" = "Medium"; +"mosaic.intensity.strong" = "Strong"; + +/* MTranServer engine errors */ +"error.mtran.http.error" = "MTranServer request failed."; +"error.mtran.http.error.recovery" = "Check your network and server status, then retry."; +"error.mtran.invalid.request" = "Invalid MTranServer request."; +"error.mtran.invalid.request.recovery" = "Check the translation configuration and retry."; +"error.mtran.invalid.url" = "Invalid MTranServer URL."; +"error.mtran.invalid.url.recovery" = "Fix the server URL in Settings."; +"error.mtran.not.available" = "MTranServer is unavailable."; +"error.mtran.not.available.recovery" = "Start MTranServer and retry."; + +/* OCR engine errors */ +"error.ocr.engine.not.available" = "OCR engine is not available."; +"error.ocr.engine.not.available.recovery" = "Select an available OCR engine in Settings."; +"error.ocr.invalid.image" = "The image is invalid and cannot be recognized."; +"error.ocr.invalid.image.recovery" = "Use a valid image and retry."; +"error.ocr.no.languages" = "No recognition language selected."; +"error.ocr.no.languages.recovery" = "Select recognition languages in Settings."; +"error.ocr.no.text.found" = "No text found in the image."; +"error.ocr.no.text.found.recovery" = "Capture an area that contains text and retry."; +"error.ocr.recognition.failed" = "Text recognition failed."; +"error.ocr.recognition.failed.recovery" = "Retry with a clearer image."; + +/* OCR language names */ +"lang.arabic" = "Arabic"; +"lang.chinese" = "Chinese"; +"lang.chinese.simplified" = "Chinese (Simplified)"; +"lang.chinese.traditional" = "Chinese (Traditional)"; +"lang.english" = "English"; +"lang.french" = "French"; +"lang.german" = "German"; +"lang.hindi" = "Hindi"; +"lang.italian" = "Italian"; +"lang.japanese" = "Japanese"; +"lang.korean" = "Korean"; +"lang.portuguese" = "Portuguese"; +"lang.russian" = "Russian"; +"lang.spanish" = "Spanish"; +"lang.thai" = "Thai"; +"lang.vietnamese" = "Vietnamese"; + +/* Preview accessibility */ +"preview.pin" = "Pin"; + +/* Translation provider recovery hints */ +"translation.provider.recovery.connectionFailed" = "Check your internet connection and retry."; +"translation.provider.recovery.emptyInput" = "The text to translate is empty."; +"translation.provider.recovery.invalidConfiguration" = "Check the translation engine configuration."; +"translation.provider.recovery.notAvailable" = "This translation engine is unavailable. Switch engines or retry later."; +"translation.provider.recovery.rateLimited" = "Too many requests. Retry later."; +"translation.provider.recovery.timeout" = "The translation request timed out. Retry."; +"translation.provider.recovery.translationFailed" = "Translation failed. Retry."; +"translation.provider.recovery.unsupportedLanguage" = "Unsupported language pair. Choose another target language."; +"translation.unknown" = "Unknown"; diff --git a/TransFrame/Resources/es.lproj/Localizable.strings b/TransFrame/Resources/es.lproj/Localizable.strings index 6c7fab7..b5e4d82 100644 --- a/TransFrame/Resources/es.lproj/Localizable.strings +++ b/TransFrame/Resources/es.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Error en el reconocimiento de texto."; "error.ocr.failed.recovery" = "Inténtelo de nuevo con una imagen más clara."; "error.ocr.no.text" = "No se reconoció texto en la imagen."; +"error.image.load.failed" = "No se pudo abrir la imagen \"%@\""; "error.ocr.no.text.recovery" = "Intente capturar un área con texto visible."; "error.ocr.cancelled" = "El reconocimiento de texto fue cancelado."; "error.ocr.server.unreachable" = "No se puede conectar al servidor OCR."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Capturar Selección"; "menu.translation.mode" = "Modo de Traducción"; "menu.translation.history" = "Historial de Traducciones"; +"menu.pick.color" = "Elegir color de la pantalla"; +"menu.open.image" = "Abrir imagen para editar..."; "menu.settings" = "Configuración..."; "menu.about" = "Acerca de TransFrame"; "menu.quit" = "Salir de TransFrame"; diff --git a/TransFrame/Resources/fr.lproj/Localizable.strings b/TransFrame/Resources/fr.lproj/Localizable.strings index 27329d9..0ce4716 100644 --- a/TransFrame/Resources/fr.lproj/Localizable.strings +++ b/TransFrame/Resources/fr.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Échec de la reconnaissance de texte."; "error.ocr.failed.recovery" = "Veuillez réessayer avec une image plus claire."; "error.ocr.no.text" = "Aucun texte n'a été reconnu dans l'image."; +"error.image.load.failed" = "Impossible d'ouvrir l'image « %@ »"; "error.ocr.no.text.recovery" = "Essayez de capturer une zone avec du texte visible."; "error.ocr.cancelled" = "La reconnaissance de texte a été annulée."; "error.ocr.server.unreachable" = "Impossible de se connecter au serveur OCR."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Capturer la sélection"; "menu.translation.mode" = "Mode Traduction"; "menu.translation.history" = "Historique des traductions"; +"menu.pick.color" = "Choisir une couleur à l'écran"; +"menu.open.image" = "Ouvrir une image à éditer..."; "menu.settings" = "Réglages..."; "menu.about" = "À propos de TransFrame"; "menu.quit" = "Quitter TransFrame"; diff --git a/TransFrame/Resources/it.lproj/Localizable.strings b/TransFrame/Resources/it.lproj/Localizable.strings index 19b2ddf..112cbbb 100644 --- a/TransFrame/Resources/it.lproj/Localizable.strings +++ b/TransFrame/Resources/it.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Riconoscimento testo fallito."; "error.ocr.failed.recovery" = "Riprova con un'immagine più chiara."; "error.ocr.no.text" = "Nessun testo riconosciuto nell'immagine."; +"error.image.load.failed" = "Impossibile aprire l'immagine \"%@\""; "error.ocr.no.text.recovery" = "Prova a catturare un'area con testo visibile."; "error.ocr.cancelled" = "Riconoscimento testo annullato."; "error.ocr.server.unreachable" = "Impossibile connettersi al server OCR."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Cattura Selezione"; "menu.translation.mode" = "Modalità Traduzione"; "menu.translation.history" = "Cronologia Traduzioni"; +"menu.pick.color" = "Preleva colore dallo schermo"; +"menu.open.image" = "Apri immagine per modificarla..."; "menu.settings" = "Impostazioni..."; "menu.about" = "Riguardo TransFrame"; "menu.quit" = "Esci da TransFrame"; diff --git a/TransFrame/Resources/ja.lproj/Localizable.strings b/TransFrame/Resources/ja.lproj/Localizable.strings index ac5fa6d..c65e4ee 100644 --- a/TransFrame/Resources/ja.lproj/Localizable.strings +++ b/TransFrame/Resources/ja.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "文字認識に失敗しました。"; "error.ocr.failed.recovery" = "より鮮明な画像でもう一度お試しください。"; "error.ocr.no.text" = "画像内でテキストが認識されませんでした。"; +"error.image.load.failed" = "画像「%@」を開けませんでした"; "error.ocr.no.text.recovery" = "テキストが表示されている範囲をキャプチャしてお試しください。"; "error.ocr.cancelled" = "文字認識がキャンセルされました。"; "error.ocr.server.unreachable" = "OCR サーバーに接続できません。"; @@ -84,6 +85,8 @@ "menu.capture.selection" = "範囲キャプチャ"; "menu.translation.mode" = "翻訳モード"; "menu.translation.history" = "翻訳履歴"; +"menu.pick.color" = "画面の色を拾う"; +"menu.open.image" = "画像を編集用に開く..."; "menu.settings" = "設定..."; "menu.about" = "TransFrame について"; "menu.quit" = "TransFrame を終了"; diff --git a/TransFrame/Resources/ko.lproj/Localizable.strings b/TransFrame/Resources/ko.lproj/Localizable.strings index 4eb4f64..f931cad 100644 --- a/TransFrame/Resources/ko.lproj/Localizable.strings +++ b/TransFrame/Resources/ko.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "텍스트 인식에 실패했습니다."; "error.ocr.failed.recovery" = "더 선명한 이미지로 다시 시도하세요."; "error.ocr.no.text" = "이미지에서 텍스트를 인식하지 못했습니다."; +"error.image.load.failed" = "이미지 \"%@\"을(를) 열 수 없습니다"; "error.ocr.no.text.recovery" = "보이는 텍스트가 있는 영역을 캡처해 보세요."; "error.ocr.cancelled" = "텍스트 인식이 취소되었습니다."; "error.ocr.server.unreachable" = "OCR 서버에 연결할 수 없습니다."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "선택 영역 캡처"; "menu.translation.mode" = "번역 모드"; "menu.translation.history" = "번역 기록"; +"menu.pick.color" = "화면 색상 선택"; +"menu.open.image" = "편집할 이미지 열기..."; "menu.settings" = "설정..."; "menu.about" = "TransFrame 정보"; "menu.quit" = "TransFrame 종료"; diff --git a/TransFrame/Resources/pt.lproj/Localizable.strings b/TransFrame/Resources/pt.lproj/Localizable.strings index c9db023..8f5fec0 100644 --- a/TransFrame/Resources/pt.lproj/Localizable.strings +++ b/TransFrame/Resources/pt.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Falha no reconhecimento de texto."; "error.ocr.failed.recovery" = "Tente novamente com uma imagem mais clara."; "error.ocr.no.text" = "Nenhum texto foi reconhecido na imagem."; +"error.image.load.failed" = "Não foi possível abrir a imagem \"%@\""; "error.ocr.no.text.recovery" = "Tente capturar uma área com texto visível."; "error.ocr.cancelled" = "O reconhecimento de texto foi cancelado."; "error.ocr.server.unreachable" = "Não é possível conectar ao servidor OCR."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Capturar Seleção"; "menu.translation.mode" = "Modo de Tradução"; "menu.translation.history" = "Histórico de Tradução"; +"menu.pick.color" = "Escolher cor da tela"; +"menu.open.image" = "Abrir imagem para edição..."; "menu.settings" = "Configurações..."; "menu.about" = "Sobre o TransFrame"; "menu.quit" = "Sair do TransFrame"; diff --git a/TransFrame/Resources/ru.lproj/Localizable.strings b/TransFrame/Resources/ru.lproj/Localizable.strings index 1d798ad..1180573 100644 --- a/TransFrame/Resources/ru.lproj/Localizable.strings +++ b/TransFrame/Resources/ru.lproj/Localizable.strings @@ -48,6 +48,7 @@ "error.ocr.failed" = "Распознавание текста не удалось."; "error.ocr.failed.recovery" = "Попробуйте снова с более чётким изображением."; "error.ocr.no.text" = "Текст на изображении не распознан."; +"error.image.load.failed" = "Не удалось открыть изображение «%@»"; "error.ocr.no.text.recovery" = "Попробуйте захватить область с видимым текстом."; "error.ocr.cancelled" = "Распознавание текста отменено."; "error.ocr.server.unreachable" = "Не удаётся подключиться к серверу OCR."; @@ -84,6 +85,8 @@ "menu.capture.selection" = "Захватить область"; "menu.translation.mode" = "Режим перевода"; "menu.translation.history" = "История переводов"; +"menu.pick.color" = "Пипетка с экрана"; +"menu.open.image" = "Открыть изображение для редактирования..."; "menu.settings" = "Настройки..."; "menu.about" = "О TransFrame"; "menu.quit" = "Выйти из TransFrame"; diff --git a/TransFrame/Resources/zh-Hans.lproj/Localizable.strings b/TransFrame/Resources/zh-Hans.lproj/Localizable.strings index 74d4b8a..0d0b252 100644 --- a/TransFrame/Resources/zh-Hans.lproj/Localizable.strings +++ b/TransFrame/Resources/zh-Hans.lproj/Localizable.strings @@ -32,6 +32,8 @@ /* 导出错误 */ "error.export.encoding.failed" = "图像编码失败。"; "error.export.encoding.failed.recovery" = "请在设置中尝试其他格式。"; +"error.image.upload.failed" = "图片上传失败:%@"; +"error.image.upload.failed.recovery" = "请检查图床配置与网络后重试。"; "error.export.encoding.failed.detail" = "无法将图像编码为 %@ 格式。"; /* 剪贴板错误 */ @@ -46,8 +48,11 @@ /* OCR 错误 */ "error.ocr.failed" = "文字识别失败。"; +"error.pipeline.ocrFailed" = "文字识别(VLM)失败:%@"; +"error.pipeline.translateFailed" = "翻译失败:%@"; "error.ocr.failed.recovery" = "请使用更清晰的图像重试。"; "error.ocr.no.text" = "图像中未识别到文字。"; +"error.image.load.failed" = "无法打开图片 “%@”"; "error.ocr.no.text.recovery" = "请尝试截取包含可见文字的区域。"; "error.ocr.cancelled" = "文字识别已取消。"; "error.ocr.server.unreachable" = "无法连接到 OCR 服务器。"; @@ -83,6 +88,8 @@ "menu.capture.fullscreen" = "全屏截图"; "menu.capture.selection" = "区域截图"; "menu.translation.mode" = "截图翻译"; +"menu.pick.color" = "屏幕取色"; +"menu.open.image" = "打开图片编辑..."; "menu.translation.history" = "翻译历史"; "menu.settings" = "设置..."; "menu.about" = "关于 TransFrame"; @@ -136,6 +143,8 @@ "preview.tooltip.confirm" = "复制到剪贴板并关闭 (回车)"; "preview.tooltip.dismiss" = "关闭 (Escape)"; "preview.tooltip.delete" = "删除选中的标注"; +"preview.tooltip.editText" = "编辑文字"; +"preview.tooltip.beautify" = "美化 (背景与阴影)"; /* 无障碍标签 */ "preview.accessibility.save" = "保存截图"; @@ -164,6 +173,8 @@ "tool.highlight" = "高亮"; "tool.mosaic" = "马赛克"; "tool.numberLabel" = "数字标签"; +"tool.privacyRedaction" = "隐私涂黑"; +"tool.calloutBubble" = "气泡标注"; /* ======================================== @@ -488,6 +499,8 @@ ======================================== */ "history.title" = "翻译历史"; +"history.filter.all" = "全部"; +"history.filter.starred" = "星标"; "history.search.placeholder" = "搜索历史记录..."; "history.clear.all" = "清除所有历史"; "history.empty.title" = "没有翻译历史"; @@ -706,6 +719,7 @@ ======================================== */ "bilingualResult.window.title" = "双语对照"; +"bilingualResult.pin" = "钉图"; "bilingualResult.loading" = "翻译中..."; "bilingualResult.loading.analyzing" = "正在分析图像..."; "bilingualResult.loading.translating" = "正在翻译文本..."; @@ -816,11 +830,6 @@ "about.acknowledgements.upstream" = "基于"; "about.acknowledgements.author.format" = "作者:%@"; "about.close" = "关闭"; -"settings.glmocr.mode" = "模式"; -"settings.glmocr.mode.cloud" = "云端"; -"settings.glmocr.mode.local" = "本地"; -"settings.glmocr.local.apiKey.optional" = "本地 MLX-VLM 服务无需 API 密钥"; -"vlm.provider.glmocr.local.description" = "本地 MLX-VLM GLM-OCR 服务"; "engine.config.fetchModels" = "获取"; "engine.config.fetchModels.failed" = "获取模型失败"; @@ -830,3 +839,89 @@ "menu.pinned.show_all" = "置顶显示所有钉图"; "tool.privacyRedaction" = "隐私遮挡"; "tool.calloutBubble" = "引线气泡"; + +/* Annotation color palette */ +"preview.color.custom" = "自定义颜色"; +"preview.color.eyedropper" = "屏幕吸管"; +"preview.fill.color" = "填充色"; + +/* Smart privacy mosaic */ +"smartMosaic.error.notConfigured" = "请先在设置中配置 VLM API Key(智能马赛克与 VLM 共用同一配置)"; +"smartMosaic.detecting" = "正在识别图中隐私..."; +"smartMosaic.noneFound" = "未检测到敏感隐私信息"; +"smartMosaic.redacted" = "已自动遮挡 %d 处隐私"; +"smartMosaic.error.failed" = "隐私识别失败,请检查网络或重试"; +"preview.tooltip.smartMosaic" = "智能马赛克"; + +/* Embedded results side panel */ +"preview.panel.ocrTab" = "文本识别"; +"preview.panel.translateTab" = "翻译"; +"preview.panel.rerun" = "重新识别"; +"preview.panel.copyAll" = "复制全部"; +"preview.panel.close" = "关闭"; +"preview.panel.expandDrawer" = "展开结果面板"; +"preview.panel.recognizing" = "正在识别..."; +"preview.panel.copyLine" = "复制"; +"preview.panel.original" = "原文"; +"preview.panel.translation" = "译文"; +"preview.panel.targetLanguage" = "目标语言"; +"preview.panel.noResults" = "暂无结果,点击「文本识别」开始"; +"preview.panel.translating" = "正在翻译..."; +"preview.panel.translate" = "翻译"; +"mosaic.intensity.light" = "轻微"; +"mosaic.intensity.medium" = "中度"; +"mosaic.intensity.strong" = "高强度"; + +/* MTranServer engine errors */ +"error.mtran.http.error" = "MTranServer 请求失败。"; +"error.mtran.http.error.recovery" = "请检查网络与服务器状态后重试。"; +"error.mtran.invalid.request" = "MTranServer 请求无效。"; +"error.mtran.invalid.request.recovery" = "请检查翻译配置后重试。"; +"error.mtran.invalid.url" = "MTranServer 地址无效。"; +"error.mtran.invalid.url.recovery" = "请在设置中修正服务器地址。"; +"error.mtran.not.available" = "MTranServer 服务不可用。"; +"error.mtran.not.available.recovery" = "请启动 MTranServer 后重试。"; + +/* OCR engine errors */ +"error.ocr.engine.not.available" = "OCR 引擎不可用。"; +"error.ocr.engine.not.available.recovery" = "请在设置中选择可用的 OCR 引擎。"; +"error.ocr.invalid.image" = "图像无效,无法识别。"; +"error.ocr.invalid.image.recovery" = "请使用有效的图像重试。"; +"error.ocr.no.languages" = "未选择识别语言。"; +"error.ocr.no.languages.recovery" = "请在设置中选择识别语言。"; +"error.ocr.no.text.found" = "未在图像中找到文字。"; +"error.ocr.no.text.found.recovery" = "请截取包含文字的区域重试。"; +"error.ocr.recognition.failed" = "文字识别失败。"; +"error.ocr.recognition.failed.recovery" = "请使用更清晰的图像重试。"; + +/* OCR language names */ +"lang.arabic" = "阿拉伯语"; +"lang.chinese" = "中文"; +"lang.chinese.simplified" = "简体中文"; +"lang.chinese.traditional" = "繁体中文"; +"lang.english" = "英语"; +"lang.french" = "法语"; +"lang.german" = "德语"; +"lang.hindi" = "印地语"; +"lang.italian" = "意大利语"; +"lang.japanese" = "日语"; +"lang.korean" = "韩语"; +"lang.portuguese" = "葡萄牙语"; +"lang.russian" = "俄语"; +"lang.spanish" = "西班牙语"; +"lang.thai" = "泰语"; +"lang.vietnamese" = "越南语"; + +/* Preview accessibility */ +"preview.pin" = "固定"; + +/* Translation provider recovery hints */ +"translation.provider.recovery.connectionFailed" = "请检查网络连接后重试。"; +"translation.provider.recovery.emptyInput" = "翻译内容为空。"; +"translation.provider.recovery.invalidConfiguration" = "请检查翻译引擎配置。"; +"translation.provider.recovery.notAvailable" = "该翻译引擎当前不可用,请切换引擎或稍后重试。"; +"translation.provider.recovery.rateLimited" = "请求过于频繁,请稍后重试。"; +"translation.provider.recovery.timeout" = "翻译请求超时,请重试。"; +"translation.provider.recovery.translationFailed" = "翻译失败,请重试。"; +"translation.provider.recovery.unsupportedLanguage" = "不支持该语言对,请更换目标语言。"; +"translation.unknown" = "未知"; diff --git a/TransFrame/Services/AnalysisImageEncoder.swift b/TransFrame/Services/AnalysisImageEncoder.swift new file mode 100644 index 0000000..094862f --- /dev/null +++ b/TransFrame/Services/AnalysisImageEncoder.swift @@ -0,0 +1,76 @@ +import CoreGraphics +import Foundation +import ImageIO +import UniformTypeIdentifiers + +/// Shared image encoder for OCR/VLM analysis requests. +/// +/// Captures come in at full Retina backing resolution. Sending them verbatim +/// multiplies upload size and vision-encoder prefill cost on OpenAI-compatible +/// endpoints, which — unlike the official OpenAI API — do not downscale large +/// images server-side. Screen text stays readable well below the limit, and +/// providers return normalized bounding boxes, so downsampling before upload +/// is safe for coordinate mapping. +enum AnalysisImageEncoder { + /// Longest edge (pixels) allowed in an encoded analysis image. + static let maxDimension = 2000 + + /// An encoded image ready for upload. + struct EncodedImage: Sendable { + let data: Data + let mimeType: String + + /// Base64 payload for `image_url`-style requests. + var base64String: String { data.base64EncodedString() } + + /// Inline data URI (`data:image/jpeg;base64,...`). + var dataURI: String { "data:\(mimeType);base64,\(base64String)" } + } + + /// Encodes the image as JPEG, downsampling first when its longest edge + /// exceeds `maxDimension`. + static func jpeg(_ image: CGImage, quality: CGFloat = 0.85) throws -> EncodedImage { + let source = downsampled(image) + guard let data = source.jpegData(quality: quality), !data.isEmpty else { + throw VLMProviderError.imageEncodingFailed + } + return EncodedImage(data: data, mimeType: "image/jpeg") + } + + /// Encodes the image as PNG, downsampling first when its longest edge + /// exceeds `maxDimension`. + static func png(_ image: CGImage) throws -> EncodedImage { + let source = downsampled(image) + guard let data = Self.encodePNG(source), !data.isEmpty else { + throw VLMProviderError.imageEncodingFailed + } + return EncodedImage(data: data, mimeType: "image/png") + } + + /// Returns `image` unchanged when it is within limits, otherwise an + /// aspect-preserving copy whose longest edge equals `maxDimension`. + static func downsampled(_ image: CGImage, maxDimension limit: Int = AnalysisImageEncoder.maxDimension) -> CGImage { + let longestEdge = max(image.width, image.height) + guard limit > 0, longestEdge > limit else { return image } + let scale = CGFloat(limit) / CGFloat(longestEdge) + let size = CGSize( + width: max(1, (CGFloat(image.width) * scale).rounded()), + height: max(1, (CGFloat(image.height) * scale).rounded()) + ) + return image.resized(to: size) ?? image + } + + private static func encodePNG(_ image: CGImage) -> Data? { + guard let data = CFDataCreateMutable(nil, 0), + let destination = CGImageDestinationCreateWithData( + data, UTType.png.identifier as CFString, 1, nil + ) else { + return nil + } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { + return nil + } + return data as Data + } +} diff --git a/TransFrame/Services/BeautifyRenderer.swift b/TransFrame/Services/BeautifyRenderer.swift new file mode 100644 index 0000000..5bed3e5 --- /dev/null +++ b/TransFrame/Services/BeautifyRenderer.swift @@ -0,0 +1,226 @@ +import AppKit +import CoreGraphics +import CoreImage + +/// Configuration for beautifying a screenshot with padding, rounded corners, soft shadows, and backgrounds. +struct BeautifyConfig: Sendable, Equatable { + enum BackgroundStyle: Sendable, Equatable { + case none + case gradient(Preset) + case solid(CodableColor) + case desktopWallpaper + + enum Preset: String, Sendable, CaseIterable { + case sunset = "Sunset" + case ocean = "Ocean" + case lavender = "Lavender" + case mint = "Mint" + case slate = "Slate" + case aurora = "Aurora" + + var colors: [NSColor] { + switch self { + case .sunset: + return [ + NSColor(srgbRed: 0.98, green: 0.45, blue: 0.41, alpha: 1.0), + NSColor(srgbRed: 0.99, green: 0.73, blue: 0.51, alpha: 1.0) + ] + case .ocean: + return [ + NSColor(srgbRed: 0.13, green: 0.43, blue: 0.88, alpha: 1.0), + NSColor(srgbRed: 0.31, green: 0.78, blue: 0.94, alpha: 1.0) + ] + case .lavender: + return [ + NSColor(srgbRed: 0.45, green: 0.35, blue: 0.85, alpha: 1.0), + NSColor(srgbRed: 0.82, green: 0.62, blue: 0.96, alpha: 1.0) + ] + case .mint: + return [ + NSColor(srgbRed: 0.16, green: 0.72, blue: 0.58, alpha: 1.0), + NSColor(srgbRed: 0.62, green: 0.89, blue: 0.75, alpha: 1.0) + ] + case .slate: + return [ + NSColor(srgbRed: 0.18, green: 0.22, blue: 0.28, alpha: 1.0), + NSColor(srgbRed: 0.35, green: 0.41, blue: 0.48, alpha: 1.0) + ] + case .aurora: + return [ + NSColor(srgbRed: 0.05, green: 0.58, blue: 0.70, alpha: 1.0), + NSColor(srgbRed: 0.48, green: 0.85, blue: 0.55, alpha: 1.0) + ] + } + } + } + } + + var isEnabled: Bool + var padding: CGFloat + var cornerRadius: CGFloat + var shadowRadius: CGFloat + var shadowOpacity: Float + var backgroundStyle: BackgroundStyle + + static let `default` = BeautifyConfig( + isEnabled: false, + padding: 36, + cornerRadius: 12, + shadowRadius: 24, + shadowOpacity: 0.35, + backgroundStyle: .gradient(.ocean) + ) + + init( + isEnabled: Bool = false, + padding: CGFloat = 36, + cornerRadius: CGFloat = 12, + shadowRadius: CGFloat = 24, + shadowOpacity: Float = 0.35, + backgroundStyle: BackgroundStyle = .gradient(.ocean) + ) { + self.isEnabled = isEnabled + self.padding = padding + self.cornerRadius = cornerRadius + self.shadowRadius = shadowRadius + self.shadowOpacity = shadowOpacity + self.backgroundStyle = backgroundStyle + } +} + +/// Hardware-accelerated CoreGraphics renderer that applies beautification packaging to screenshots. +final class BeautifyRenderer: @unchecked Sendable { + static let shared = BeautifyRenderer() + + private init() {} + + /// Renders the input image wrapped with background, shadow, padding, and corner radius. + func render(image: CGImage, config: BeautifyConfig, screen: NSScreen? = nil) -> CGImage? { + guard config.isEnabled else { return image } + + let originalWidth = CGFloat(image.width) + let originalHeight = CGFloat(image.height) + let padding = config.padding + let targetWidth = originalWidth + padding * 2 + let targetHeight = originalHeight + padding * 2 + + guard let colorSpace = image.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB) else { return nil } + guard let context = CGContext( + data: nil, + width: Int(targetWidth), + height: Int(targetHeight), + bitsPerComponent: 8, + bytesPerRow: 0, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + return nil + } + + let fullRect = CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight) + + // 1. Draw Background + drawBackground(config.backgroundStyle, in: context, rect: fullRect, screen: screen) + + // 2. Draw Rounded Image with Shadow + let imageRect = CGRect(x: padding, y: padding, width: originalWidth, height: originalHeight) + + // Rounded path shared by the shadow pass and the clipped image pass. + let path = CGPath( + roundedRect: imageRect, + cornerWidth: config.cornerRadius, + cornerHeight: config.cornerRadius, + transform: nil + ) + + // Shadow pass: fill the path with the shadow applied, then restore. + // Filling before clipping keeps the shadow outside the rounded rect — + // a single clip-then-draw pass would clip the shadow away entirely. + if config.shadowOpacity > 0 && config.shadowRadius > 0 { + let shadowColor = NSColor.black.withAlphaComponent(CGFloat(config.shadowOpacity)).cgColor + context.saveGState() + context.setShadow( + offset: CGSize(width: 0, height: -config.shadowRadius * 0.4), + blur: config.shadowRadius, + color: shadowColor + ) + context.addPath(path) + context.setFillColor(NSColor.black.cgColor) + context.fillPath() + context.restoreGState() + } + + // Image pass: clip to the rounded rect and draw the screenshot. + context.saveGState() + context.addPath(path) + context.clip() + context.draw(image, in: imageRect) + context.restoreGState() + + return context.makeImage() + } + + private func drawBackground(_ style: BeautifyConfig.BackgroundStyle, in context: CGContext, rect: CGRect, screen: NSScreen?) { + switch style { + case .none: + break + + case .solid(let color): + context.setFillColor(color.cgColor) + context.fill(rect) + + case .gradient(let preset): + let colors = preset.colors.map { $0.cgColor } as CFArray + let colorSpace = context.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB)! + if let gradient = CGGradient(colorsSpace: colorSpace, colors: colors, locations: [0.0, 1.0]) { + context.drawLinearGradient( + gradient, + start: CGPoint(x: rect.minX, y: rect.maxY), + end: CGPoint(x: rect.maxX, y: rect.minY), + options: [] + ) + } + + case .desktopWallpaper: + if let wallpaperImage = fetchBlurredWallpaper(for: screen) { + context.draw(wallpaperImage, in: rect) + } else { + // Fallback to default gradient + drawBackground(.gradient(.slate), in: context, rect: rect, screen: screen) + } + } + } + + private func fetchBlurredWallpaper(for screen: NSScreen?) -> CGImage? { + let wallpaperURL: URL? + if let screen { + wallpaperURL = NSWorkspace.shared.desktopImageURL(for: screen) + } else if Thread.isMainThread { + wallpaperURL = MainActor.assumeIsolated { + guard let mainScreen = NSScreen.main else { return nil } + return NSWorkspace.shared.desktopImageURL(for: mainScreen) + } + } else { + wallpaperURL = nil + } + + guard let wallpaperURL, + let nsImage = NSImage(contentsOf: wallpaperURL) else { + return nil + } + var proposed = NSRect(origin: .zero, size: nsImage.size) + guard let cg = nsImage.cgImage(forProposedRect: &proposed, context: nil, hints: nil) else { + return nil + } + + // Apply slight Gaussian blur + let ciImage = CIImage(cgImage: cg) + guard let filter = CIFilter(name: "CIGaussianBlur") else { return cg } + filter.setValue(ciImage, forKey: kCIInputImageKey) + filter.setValue(30.0, forKey: kCIInputRadiusKey) + guard let output = filter.outputImage else { return cg } + + let ciContext = CIContext() + return ciContext.createCGImage(output, from: ciImage.extent) ?? cg + } +} diff --git a/TransFrame/Services/ClaudeVLMProvider.swift b/TransFrame/Services/ClaudeVLMProvider.swift index 72689e6..d1fe46a 100644 --- a/TransFrame/Services/ClaudeVLMProvider.swift +++ b/TransFrame/Services/ClaudeVLMProvider.swift @@ -88,18 +88,20 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { private let maxContinuationAttempts = 3 func analyze(image: CGImage) async throws -> ScreenAnalysisResult { - guard let imageData = image.jpegData(quality: 0.85), !imageData.isEmpty else { - throw VLMProviderError.imageEncodingFailed - } + try await analyze(image: image, prompt: .ocr) + } - let base64Image = imageData.base64EncodedString() + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult { + let encoded = try AnalysisImageEncoder.jpeg(image) let imageSize = CGSize(width: image.width, height: image.height) // Use multi-turn conversation with continuation support let vlmResponse = try await analyzeWithContinuation( - base64Image: base64Image, + base64Image: encoded.base64String, + mediaType: encoded.mimeType, imageSize: imageSize, - maxAttempts: maxContinuationAttempts + maxAttempts: maxContinuationAttempts, + prompts: prompt ) return vlmResponse.toScreenAnalysisResult(imageSize: imageSize) @@ -108,8 +110,10 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { /// Performs analysis with automatic continuation on truncation private func analyzeWithContinuation( base64Image: String, + mediaType: String = "image/jpeg", imageSize: CGSize, - maxAttempts: Int + maxAttempts: Int, + prompts: VLMPromptPair = .ocr ) async throws -> VLMAnalysisResponse { var allSegments: [VLMTextSegment] = [] var conversationHistory: [ClaudeMessage] = [ @@ -119,11 +123,11 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { .image(ClaudeImageContent( source: ClaudeImageSource( type: "base64", - mediaType: "image/jpeg", + mediaType: mediaType, data: base64Image ) )), - .text(VLMPromptTemplate.userPrompt), + .text(prompts.user), ] ), ] @@ -131,7 +135,8 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { for attempt in 0.. 0 + isContinuation: attempt > 0, + systemPrompt: prompts.system ) let responseData = try await executeRequest(request) @@ -245,8 +250,13 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { return (content, isTruncated, claudeResponse.stopReason) } + /// Placeholder replacing the base64 image in continuation rounds, so the + /// same capture is not uploaded again for every truncation retry. + private static let continuationImagePlaceholder = + "(image omitted: the same image as in the first message)" + /// Builds request with custom messages and continuation settings - private func buildRequest(messages: [ClaudeMessage], isContinuation: Bool) throws -> URLRequest { + private func buildRequest(messages: [ClaudeMessage], isContinuation: Bool, systemPrompt: String = VLMPromptTemplate.systemPrompt) throws -> URLRequest { let endpoint = configuration.baseURL.resolvingLocalhost.appendingPathComponent("v1/messages") var request = URLRequest(url: endpoint) @@ -259,14 +269,32 @@ struct ClaudeVLMProvider: VLMProvider, Sendable { request.setValue(Self.apiVersion, forHTTPHeaderField: "anthropic-version") request.timeoutInterval = timeout + // Continuation rounds keep the conversation shape but drop the base64 + // image payload: the assistant's partial JSON already anchors the + // context, and re-uploading would multiply bandwidth and prompt tokens. + let effectiveMessages: [ClaudeMessage] + if isContinuation { + effectiveMessages = messages.map { message in + let blocks = message.content.map { block -> ClaudeContentBlock in + if case .image = block { + return .text(Self.continuationImagePlaceholder) + } + return block + } + return ClaudeMessage(role: message.role, content: blocks) + } + } else { + effectiveMessages = messages + } + // Use higher max_tokens for continuation requests to minimize truncation let maxTokens = isContinuation ? 16384 : 8192 let requestBody = ClaudeMessagesRequest( model: configuration.modelName, maxTokens: maxTokens, - system: VLMPromptTemplate.systemPrompt, - messages: messages + system: systemPrompt, + messages: effectiveMessages ) let encoder = JSONEncoder() diff --git a/TransFrame/Services/DocumentRenderer.swift b/TransFrame/Services/DocumentRenderer.swift index cf537a3..e384c53 100644 --- a/TransFrame/Services/DocumentRenderer.swift +++ b/TransFrame/Services/DocumentRenderer.swift @@ -139,17 +139,21 @@ final class DocumentRenderer: @unchecked Sendable { context.saveGState() // 1. Draw pointer tail from bubble to target point + let attachX = max(cgBubbleRect.minX + 10, min(cgBubbleRect.maxX - 10, cgTarget.x)) + let attachY = cgTarget.y < cgBubbleRect.minY ? cgBubbleRect.minY : (cgTarget.y > cgBubbleRect.maxY ? cgBubbleRect.maxY : cgBubbleRect.midY) + let halfBase = min(8.0, max(4.0, cgBubbleRect.width * 0.1)) let tailPath = CGMutablePath() tailPath.move(to: cgTarget) - // Find attachment point - let attachPoint = CGPoint( - x: max(cgBubbleRect.minX + 8, min(cgBubbleRect.maxX - 8, cgTarget.x)), - y: cgTarget.y < cgBubbleRect.minY ? cgBubbleRect.minY : cgBubbleRect.maxY - ) - tailPath.addLine(to: attachPoint) + tailPath.addLine(to: CGPoint(x: attachX - halfBase, y: attachY)) + tailPath.addLine(to: CGPoint(x: attachX + halfBase, y: attachY)) + tailPath.closeSubpath() + context.setFillColor(annotation.bubbleColor.cgColor) + context.addPath(tailPath) + context.fillPath() context.setStrokeColor(annotation.bubbleColor.cgColor) context.setLineWidth(2.0) - context.addPath(tailPath) + context.move(to: cgTarget) + context.addLine(to: CGPoint(x: attachX, y: attachY)) context.strokePath() // 2. Draw bubble body @@ -193,7 +197,7 @@ final class DocumentRenderer: @unchecked Sendable { context.saveGState() if annotation.isFilled { - context.setFillColor(annotation.style.color.cgColor) + context.setFillColor(annotation.effectiveFillColor.cgColor) context.fill(rect) } else { context.setStrokeColor(annotation.style.color.cgColor) @@ -219,7 +223,7 @@ final class DocumentRenderer: @unchecked Sendable { context.saveGState() if annotation.isFilled { - context.setFillColor(annotation.style.color.cgColor) + context.setFillColor(annotation.effectiveFillColor.cgColor) context.fillEllipse(in: rect) } else { context.setStrokeColor(annotation.style.color.cgColor) @@ -300,25 +304,33 @@ final class DocumentRenderer: @unchecked Sendable { ) { let start = annotation.startPoint.applying(transform) let end = annotation.endPoint.applying(transform) + let control = annotation.controlPoint.map { $0.applying(transform) } let cgStart = CGPoint(x: start.x, y: targetHeight - start.y) let cgEnd = CGPoint(x: end.x, y: targetHeight - end.y) + let cgControl = control.map { CGPoint(x: $0.x, y: targetHeight - $0.y) } context.saveGState() context.setStrokeColor(annotation.style.color.cgColor) context.setFillColor(annotation.style.color.cgColor) context.setLineWidth(annotation.style.lineWidth) context.setLineCap(.round) + context.setLineJoin(.round) - // Draw shaft + // Draw shaft (curved if controlPoint is set) context.beginPath() context.move(to: cgStart) - context.addLine(to: cgEnd) + if let cp = cgControl { + context.addQuadCurve(to: cgEnd, control: cp) + } else { + context.addLine(to: cgEnd) + } context.strokePath() - // Draw arrowhead - let dx = cgEnd.x - cgStart.x - let dy = cgEnd.y - cgStart.y + // Draw arrowhead aligned with tangent at end + let prevPoint = cgControl ?? cgStart + let dx = cgEnd.x - prevPoint.x + let dy = cgEnd.y - prevPoint.y let angle = atan2(dy, dx) let arrowLength = max(annotation.style.lineWidth * 4, 14.0) let arrowAngle = CGFloat.pi / 6 @@ -409,19 +421,35 @@ final class DocumentRenderer: @unchecked Sendable { targetHeight: CGFloat, transform: CGAffineTransform ) { - let pos = annotation.position.applying(transform) + guard !annotation.content.isEmpty else { return } + let fontSize = max(10, annotation.style.fontSize) - let font = NSFont.systemFont(ofSize: fontSize, weight: .regular) + let font = NSFont(name: annotation.style.fontName, size: fontSize) + ?? NSFont.systemFont(ofSize: fontSize, weight: .regular) + let paragraph = NSMutableParagraphStyle() + paragraph.lineBreakMode = .byWordWrapping let attributes: [NSAttributedString.Key: Any] = [ .font: font, + .paragraphStyle: paragraph, .foregroundColor: NSColor(cgColor: annotation.style.color.cgColor) ?? .black ] let attrString = NSAttributedString(string: annotation.content, attributes: attributes) - let line = CTLineCreateWithAttributedString(attrString) + + // Frame in the measured bounds so wrapped lines and hard line breaks both fit + let transformedBounds = annotation.bounds.applying(transform) + let frameRect = CGRect( + x: transformedBounds.minX, + y: targetHeight - transformedBounds.maxY, + width: transformedBounds.width, + height: transformedBounds.height + ) + guard frameRect.width > 0, frameRect.height > 0 else { return } context.saveGState() - context.textPosition = CGPoint(x: pos.x, y: targetHeight - pos.y - fontSize) - CTLineDraw(line, context) + let framesetter = CTFramesetterCreateWithAttributedString(attrString) + let path = CGPath(rect: frameRect, transform: nil) + let frame = CTFramesetterCreateFrame(framesetter, CFRangeMake(0, attrString.length), path, nil) + CTFrameDraw(frame, context) context.restoreGState() } @@ -432,15 +460,38 @@ final class DocumentRenderer: @unchecked Sendable { transform: CGAffineTransform ) { let pos = annotation.position.applying(transform) + let cgCenter = CGPoint(x: pos.x, y: targetHeight - pos.y) let radius = annotation.size / 2.0 let rect = CGRect( - x: pos.x - radius, - y: targetHeight - pos.y - radius, + x: cgCenter.x - radius, + y: cgCenter.y - radius, width: annotation.size, height: annotation.size ) context.saveGState() + + if let lp = annotation.leadPoint?.applying(transform) { + let cgLead = CGPoint(x: lp.x, y: targetHeight - lp.y) + let lineWidth = max(2.0, annotation.size / 12.0) + context.setStrokeColor(annotation.color.cgColor) + context.setLineWidth(lineWidth) + context.setLineCap(.round) + context.beginPath() + context.move(to: cgCenter) + context.addLine(to: cgLead) + context.strokePath() + + let dotRadius = max(3.5, annotation.size * 0.14) + context.setFillColor(annotation.color.cgColor) + context.fillEllipse(in: CGRect( + x: cgLead.x - dotRadius, + y: cgLead.y - dotRadius, + width: dotRadius * 2, + height: dotRadius * 2 + )) + } + context.setFillColor(annotation.color.cgColor) context.fillEllipse(in: rect) diff --git a/TransFrame/Services/GLMOCRVLMProvider.swift b/TransFrame/Services/GLMOCRVLMProvider.swift deleted file mode 100644 index eec74ff..0000000 --- a/TransFrame/Services/GLMOCRVLMProvider.swift +++ /dev/null @@ -1,528 +0,0 @@ -// -// GLMOCRVLMProvider.swift -// TransFrame -// -// Integrates Zhipu GLM-OCR layout parsing for screenshot text extraction. -// - -import CoreGraphics -import Foundation - -struct GLMOCRVLMProvider: VLMProvider, Sendable { - let id: String = "glm_ocr" - let name: String = "GLM OCR" - let configuration: VLMProviderConfiguration - let mode: GLMOCRMode - - static let defaultBaseURL: URL = { - guard let url = URL(string: "https://open.bigmodel.cn/api/paas/v4") else { - fatalError("Invalid URL literal for GLMOCRVLMProvider.defaultBaseURL") - } - return url - }() - static let defaultModel = "glm-ocr" - static let defaultLocalBaseURL: URL = { - guard let url = URL(string: "http://127.0.0.1:18081") else { - fatalError("Invalid URL literal for GLMOCRVLMProvider.defaultLocalBaseURL") - } - return url - }() - static let defaultLocalModel = "mlx-community/GLM-OCR-bf16" - - static let connectionTestImageDataURI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAE0lEQVR4nGP8//8/AwwwwVl4OQCWbgMF7ZjH1AAAAABJRU5ErkJggg==" - - private let timeout: TimeInterval - - init(configuration: VLMProviderConfiguration, mode: GLMOCRMode = .cloud, timeout: TimeInterval = 60) { - self.configuration = configuration - self.mode = mode - self.timeout = timeout - } - - var isAvailable: Bool { - get async { - switch mode { - case .cloud: - return !configuration.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - case .local: - return true - } - } - } - - func analyze(image: CGImage) async throws -> ScreenAnalysisResult { - guard let imageData = image.jpegData(quality: 0.9), !imageData.isEmpty else { - throw VLMProviderError.imageEncodingFailed - } - - let imageSize = CGSize(width: image.width, height: image.height) - let dataURI = "data:image/jpeg;base64,\(imageData.base64EncodedString())" - - switch mode { - case .cloud: - let request = try Self.makeLayoutParsingRequest( - baseURL: configuration.baseURL, - apiKey: configuration.apiKey, - modelName: configuration.modelName, - fileDataURI: dataURI, - timeout: timeout - ) - let data = try await executeRequest(request, mode: .cloud) - return try Self.parseResponse(data, fallbackImageSize: imageSize) - case .local: - let request = try Self.makeLocalChatRequest( - baseURL: configuration.baseURL, - apiKey: configuration.apiKey, - modelName: configuration.modelName, - fileDataURI: dataURI, - timeout: timeout - ) - let data = try await executeRequest(request, mode: .local) - return try Self.parseLocalResponse(data, fallbackImageSize: imageSize) - } - } - - static func makeLayoutParsingRequest( - baseURL: URL, - apiKey: String, - modelName: String, - fileDataURI: String, - timeout: TimeInterval - ) throws -> URLRequest { - let trimmedAPIKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmedAPIKey.isEmpty else { - throw VLMProviderError.invalidConfiguration("GLM OCR requires an API key.") - } - - let endpoint = baseURL.resolvingLocalhost.appendingPathComponent("layout_parsing") - var request = URLRequest(url: endpoint) - request.httpMethod = "POST" - request.timeoutInterval = timeout - request.setValue("Bearer \(trimmedAPIKey)", forHTTPHeaderField: "Authorization") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - - let body = GLMOCRLayoutParsingRequest( - model: modelName.isEmpty ? defaultModel : modelName, - file: fileDataURI, - returnCropImages: false, - needLayoutVisualization: false - ) - request.httpBody = try JSONEncoder().encode(body) - return request - } - - static func makeLocalChatRequest( - baseURL: URL, - apiKey: String, - modelName: String, - fileDataURI: String, - timeout: TimeInterval - ) throws -> URLRequest { - let endpoint = baseURL.resolvingLocalhost.appendingPathComponent("chat/completions") - var request = URLRequest(url: endpoint) - request.httpMethod = "POST" - request.timeoutInterval = timeout - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - - let body = GLMOCRLocalChatRequest( - model: modelName.isEmpty ? defaultLocalModel : modelName, - messages: [ - GLMOCRLocalChatMessage( - role: "system", - content: .text(VLMPromptTemplate.localModelSystemPrompt) - ), - GLMOCRLocalChatMessage( - role: "user", - content: .vision([ - .text(VLMPromptTemplate.localModelUserPrompt + "\nReturn only valid JSON."), - .imageURL(GLMOCRLocalImageURL(url: fileDataURI)) - ]) - ), - ], - maxTokens: 4096, - temperature: 0.1 - ) - request.httpBody = try JSONEncoder().encode(body) - return request - } - - static func parseResponse(_ data: Data, fallbackImageSize: CGSize) throws -> ScreenAnalysisResult { - let decoder = JSONDecoder() - decoder.keyDecodingStrategy = .convertFromSnakeCase - - if let errorResponse = try? decoder.decode(GLMOCRAPIErrorResponse.self, from: data), - let message = errorResponse.error.message, - !message.isEmpty { - throw VLMProviderError.invalidResponse(message) - } - - let response: GLMOCRLayoutParsingResponse - do { - response = try decoder.decode(GLMOCRLayoutParsingResponse.self, from: data) - } catch { - throw VLMProviderError.parsingFailed("Failed to decode GLM OCR response: \(error.localizedDescription)") - } - - let segments = response.layoutDetails - .flatMap { $0 } - .compactMap { item in - textSegment(from: item) - } - - let resolvedImageSize = response.dataInfo?.pages.first.map { - CGSize(width: $0.width, height: $0.height) - } ?? fallbackImageSize - - return ScreenAnalysisResult(segments: segments, imageSize: resolvedImageSize) - } - - static func parseLocalResponse(_ data: Data, fallbackImageSize: CGSize) throws -> ScreenAnalysisResult { - let decoder = JSONDecoder() - decoder.keyDecodingStrategy = .convertFromSnakeCase - - if let errorResponse = try? decoder.decode(GLMOCRLocalErrorResponse.self, from: data) { - throw VLMProviderError.invalidResponse(errorResponse.error.message) - } - - let response: GLMOCRLocalChatResponse - do { - response = try decoder.decode(GLMOCRLocalChatResponse.self, from: data) - } catch { - throw VLMProviderError.parsingFailed("Failed to decode local GLM OCR response: \(error.localizedDescription)") - } - - guard let content = response.choices?.first?.message?.content, - !content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw VLMProviderError.invalidResponse("No content in local GLM OCR response") - } - - do { - return try parseLocalContent(content).toScreenAnalysisResult(imageSize: fallbackImageSize) - } catch { - throw VLMProviderError.parsingFailed("Failed to parse local GLM OCR content: \(error.localizedDescription)") - } - } - - private func executeRequest(_ request: URLRequest, mode: GLMOCRMode) async throws -> Data { - do { - let (data, response) = try await URLSession.shared.data(for: request) - guard let httpResponse = response as? HTTPURLResponse else { - throw VLMProviderError.invalidResponse("Invalid HTTP response") - } - - switch httpResponse.statusCode { - case 200: - return data - case 401, 403: - throw VLMProviderError.authenticationFailed - case 429: - throw VLMProviderError.rateLimited( - retryAfter: httpResponse.value(forHTTPHeaderField: "Retry-After").flatMap(TimeInterval.init), - message: parseAPIErrorMessage(from: data, mode: mode) - ) - default: - let message = parseAPIErrorMessage(from: data, mode: mode) ?? "HTTP \(httpResponse.statusCode)" - throw VLMProviderError.invalidResponse(message) - } - } catch let error as VLMProviderError { - throw error - } catch { - throw VLMProviderError.networkError(error.localizedDescription) - } - } - - private func parseAPIErrorMessage(from data: Data, mode: GLMOCRMode) -> String? { - let decoder = JSONDecoder() - decoder.keyDecodingStrategy = .convertFromSnakeCase - switch mode { - case .cloud: - if let response = try? decoder.decode(GLMOCRAPIErrorResponse.self, from: data), - let message = response.error.message, - !message.isEmpty { - return message - } - case .local: - if let response = try? decoder.decode(GLMOCRLocalErrorResponse.self, from: data) { - return response.error.message - } - } - return String(data: data, encoding: .utf8) - } - - private static func parseLocalContent(_ content: String) throws -> VLMAnalysisResponse { - let cleanedContent = extractJSON(from: content) - - if let jsonData = cleanedContent.data(using: .utf8), - let response = try? JSONDecoder().decode(VLMAnalysisResponse.self, from: jsonData) { - return response - } - - if let jsonData = cleanedContent.data(using: .utf8), - let textOnlyResponse = try? JSONDecoder().decode(GLMOCRLocalTextOnlyResponse.self, from: jsonData), - let textContent = textOnlyResponse.resolvedText, - let plainTextResponse = parsePlainTextResponse(textContent) { - return plainTextResponse - } - - if let plainTextResponse = parsePlainTextResponse(content) { - return plainTextResponse - } - - throw VLMProviderError.parsingFailed("Content was not valid JSON") - } - - private static func textSegment(from item: GLMOCRLayoutItem) -> TextSegment? { - let text = cleanedContent(from: item) - guard !text.isEmpty else { - return nil - } - - guard item.bbox2D.count == 4 else { - return nil - } - - let x1 = clamp(item.bbox2D[0]) - let y1 = clamp(item.bbox2D[1]) - let x2 = clamp(item.bbox2D[2]) - let y2 = clamp(item.bbox2D[3]) - let width = max(0, x2 - x1) - let height = max(0, y2 - y1) - - guard width > 0, height > 0 else { - return nil - } - - return TextSegment( - text: text, - boundingBox: CGRect(x: x1, y: y1, width: width, height: height), - confidence: 1.0 - ) - } - - private static func cleanedContent(from item: GLMOCRLayoutItem) -> String { - let trimmed = item.content.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { - return "" - } - - switch item.label { - case "image": - return "" - case "table": - return stripHTML(from: trimmed) - default: - return collapseWhitespace(in: trimmed) - } - } - - private static func stripHTML(from string: String) -> String { - let withoutTags = string.replacingOccurrences( - of: "<[^>]+>", - with: " ", - options: .regularExpression - ) - return collapseWhitespace(in: withoutTags) - } - - private static func collapseWhitespace(in string: String) -> String { - string.replacingOccurrences( - of: "\\s+", - with: " ", - options: .regularExpression - ).trimmingCharacters(in: .whitespacesAndNewlines) - } - - private static func clamp(_ value: CGFloat) -> CGFloat { - min(max(value, 0), 1) - } - - private static func extractJSON(from content: String) -> String { - VLMJSONExtractor.extractJSON(from: content) - } - - private static func parsePlainTextResponse(_ content: String) -> VLMAnalysisResponse? { - let lines = content - .trimmingCharacters(in: .whitespacesAndNewlines) - .components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } - .filter { line in - guard !line.isEmpty else { return false } - if line == "```" || line.hasPrefix("```") { return false } - if ["{", "}", "[", "]"].contains(line) { return false } - return true - } - - guard !lines.isEmpty else { - return nil - } - - let segments = lines.enumerated().map { index, text in - VLMTextSegment( - text: text, - boundingBox: VLMBoundingBox( - x: 0, - y: CGFloat(index) / CGFloat(max(lines.count, 1)), - width: 1, - height: 1 / CGFloat(max(lines.count, 1)) - ), - confidence: nil - ) - } - - return VLMAnalysisResponse(segments: segments) - } -} - -private struct GLMOCRLayoutParsingRequest: Encodable, Sendable { - let model: String - let file: String - let returnCropImages: Bool - let needLayoutVisualization: Bool - - enum CodingKeys: String, CodingKey { - case model - case file - case returnCropImages = "return_crop_images" - case needLayoutVisualization = "need_layout_visualization" - } -} - -private struct GLMOCRLocalChatRequest: Encodable, Sendable { - let model: String - let messages: [GLMOCRLocalChatMessage] - let maxTokens: Int - let temperature: Double - - enum CodingKeys: String, CodingKey { - case model, messages, temperature - case maxTokens = "max_tokens" - } -} - -private struct GLMOCRLocalTextOnlyResponse: Decodable, Sendable { - let text: String? - let Text: String? - - var resolvedText: String? { - text ?? Text - } -} - -private struct GLMOCRLocalChatMessage: Encodable, Sendable { - let role: String - let content: MessageContent - - enum MessageContent: Sendable { - case text(String) - case vision([VisionContent]) - } - - enum VisionContent: Sendable { - case text(String) - case imageURL(GLMOCRLocalImageURL) - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(role, forKey: .role) - - switch content { - case .text(let text): - try container.encode(text, forKey: .content) - case .vision(let contents): - var contentArray = container.nestedUnkeyedContainer(forKey: .content) - for item in contents { - switch item { - case .text(let text): - try contentArray.encode(["type": "text", "text": text]) - case .imageURL(let imageURL): - var itemContainer = contentArray.nestedContainer(keyedBy: VisionCodingKeys.self) - try itemContainer.encode("image_url", forKey: .type) - try itemContainer.encode(imageURL, forKey: .imageUrl) - } - } - } - } - - enum CodingKeys: String, CodingKey { - case role, content - } - - enum VisionCodingKeys: String, CodingKey { - case type - case imageUrl = "image_url" - } -} - -private struct GLMOCRLocalImageURL: Encodable, Sendable { - let url: String - let detail: String - - init(url: String, detail: String = "high") { - self.url = url - self.detail = detail - } -} - -struct GLMOCRLayoutParsingResponse: Decodable, Sendable { - let id: String? - let model: String? - let mdResults: String? - let layoutDetails: [[GLMOCRLayoutItem]] - let dataInfo: GLMOCRDataInfo? -} - -struct GLMOCRLayoutItem: Decodable, Sendable { - let index: Int? - let label: String - let bbox2D: [CGFloat] - let content: String - let height: Int? - let width: Int? -} - -struct GLMOCRDataInfo: Decodable, Sendable { - let numPages: Int? - let pages: [GLMOCRPageInfo] -} - -struct GLMOCRPageInfo: Decodable, Sendable { - let width: CGFloat - let height: CGFloat -} - -private struct GLMOCRAPIErrorResponse: Decodable, Sendable { - let error: GLMOCRAPIErrorBody -} - -private struct GLMOCRAPIErrorBody: Decodable, Sendable { - let message: String? -} - -struct GLMOCRLocalModelsResponse: Decodable, Sendable { - let data: [GLMOCRLocalModel] -} - -struct GLMOCRLocalModel: Decodable, Sendable { - let id: String -} - -private struct GLMOCRLocalChatResponse: Decodable, Sendable { - let choices: [GLMOCRLocalChatChoice]? -} - -private struct GLMOCRLocalChatChoice: Decodable, Sendable { - let message: GLMOCRLocalResponseMessage? -} - -private struct GLMOCRLocalResponseMessage: Decodable, Sendable { - let content: String? -} - -private struct GLMOCRLocalErrorResponse: Decodable, Sendable { - let error: GLMOCRLocalErrorBody -} - -private struct GLMOCRLocalErrorBody: Decodable, Sendable { - let message: String -} diff --git a/TransFrame/Services/GeminiVLMProvider.swift b/TransFrame/Services/GeminiVLMProvider.swift index 14dbb57..1187707 100644 --- a/TransFrame/Services/GeminiVLMProvider.swift +++ b/TransFrame/Services/GeminiVLMProvider.swift @@ -24,14 +24,14 @@ final class GeminiVLMProvider: VLMProvider, Sendable { } func analyze(image: CGImage) async throws -> ScreenAnalysisResult { - guard let imageData = image.jpegData(quality: 0.85), !imageData.isEmpty else { - throw VLMProviderError.imageEncodingFailed - } + try await analyze(image: image, prompt: .ocr) + } - let base64Image = imageData.base64EncodedString() + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult { + let encoded = try AnalysisImageEncoder.jpeg(image) let imageSize = CGSize(width: image.width, height: image.height) - let request = try buildRequest(base64Image: base64Image) + let request = try buildRequest(base64Image: encoded.base64String, prompts: prompt) let responseData = try await executeRequest(request) let content = try extractContent(from: responseData) @@ -49,7 +49,7 @@ final class GeminiVLMProvider: VLMProvider, Sendable { } } - private func buildRequest(base64Image: String) throws -> URLRequest { + private func buildRequest(base64Image: String, prompts: VLMPromptPair = .ocr) throws -> URLRequest { let baseURL = configuration.baseURL.resolvingLocalhost let model = configuration.modelName.isEmpty ? "gemini-2.0-flash" : configuration.modelName @@ -86,16 +86,17 @@ final class GeminiVLMProvider: VLMProvider, Sendable { [ "role": "user", "content": [ - ["type": "text", "text": VLMPromptTemplate.userPrompt], + ["type": "text", "text": prompts.user], ["type": "image_url", "image_url": ["url": "data:image/jpeg;base64,\(base64Image)"]] ] ] - ] + ], + "response_format": ["type": "json_object"] ] request.httpBody = try JSONSerialization.data(withJSONObject: body) } else { // Gemini generateContent payload - let prompt = "\(VLMPromptTemplate.systemPrompt)\n\n\(VLMPromptTemplate.userPrompt)" + let prompt = "\(prompts.system)\n\n\(prompts.user)" let body: [String: Any] = [ "contents": [ [ diff --git a/TransFrame/Services/HistoryStore.swift b/TransFrame/Services/HistoryStore.swift index 2e9314c..aecad88 100644 --- a/TransFrame/Services/HistoryStore.swift +++ b/TransFrame/Services/HistoryStore.swift @@ -33,14 +33,32 @@ final class HistoryStore: ObservableObject { /// The current search query @Published private(set) var searchQuery: String = "" - /// Filtered entries based on search query + /// Active category filter (nil = all categories) + @Published var selectedCategory: HistoryCategory? = nil { + didSet { updateFilteredEntries() } + } + + /// Whether to only show starred favorites + @Published var onlyFavorites: Bool = false { + didSet { updateFilteredEntries() } + } + + /// Filtered entries based on search query and category @Published private(set) var filteredEntries: [TranslationHistory] = [] /// Whether more entries can be loaded @Published private(set) var hasMoreEntries: Bool = false /// Number of entries currently displayed - @Published private(set) var displayedCount: Int = 50 + @Published private(set) var displayedCount: Int = 100 + + /// Block-based notification observer tokens; removed on deinit because + /// HistoryStore is recreated per window open and NotificationCenter + /// otherwise keeps the closures (and duplicate handlers) alive forever. + /// `nonisolated(unsafe)`: written only from the MainActor init path, and + /// read in deinit when no concurrent writer can exist; + /// `removeObserver` itself is thread-safe. + nonisolated(unsafe) private var notificationTokens: [NSObjectProtocol] = [] // MARK: - Initialization @@ -49,6 +67,80 @@ final class HistoryStore: ObservableObject { self.historyKey = historyKey loadHistory() updateFilteredEntries() + setupNotificationObservers() + } + + deinit { + notificationTokens.forEach(NotificationCenter.default.removeObserver) + } + + private func setupNotificationObservers() { + notificationTokens.append(NotificationCenter.default.addObserver( + forName: .colorDidPick, + object: nil, + queue: .main + ) { [weak self] notif in + guard let hex = notif.object as? String else { return } + Task { @MainActor in + let entry = TranslationHistory( + sourceText: "Color Picked", + translatedText: hex, + sourceLanguage: "RGB Hex", + targetLanguage: hex, + category: .color + ) + self?.addEntry(entry) + } + }) + + notificationTokens.append(NotificationCenter.default.addObserver( + forName: .imageDidUpload, + object: nil, + queue: .main + ) { [weak self] notif in + guard let result = notif.object as? UploadResult else { return } + Task { @MainActor in + let entry = TranslationHistory( + sourceText: result.markdown, + translatedText: result.url, + sourceLanguage: result.provider.displayName, + targetLanguage: "Public URL", + category: .upload + ) + self?.addEntry(entry) + } + }) + } + + /// Toggles favorite status for an entry + func toggleFavorite(for id: UUID) { + guard let index = entries.firstIndex(where: { $0.id == id }) else { return } + entries[index].isFavorite.toggle() + saveHistory() + updateFilteredEntries() + } + + /// Adds a general history entry + func addEntry(_ entry: TranslationHistory) { + // Deduplicate only on fully identical content, and carry over the + // previous favorite flag — silently dropping a starred entry (the old + // translatedText+category match) lost user state without notice. + if let index = entries.firstIndex(where: { + $0.sourceText == entry.sourceText && + $0.translatedText == entry.translatedText && + $0.category == entry.category + }) { + var merged = entry + merged.isFavorite = entries[index].isFavorite + entries[index] = merged + } else { + entries.insert(entry, at: 0) + } + if entries.count > Self.maxHistoryEntries { + entries = Array(entries.prefix(Self.maxHistoryEntries)) + } + saveHistory() + updateFilteredEntries() } // MARK: - Public API @@ -178,18 +270,28 @@ final class HistoryStore: ObservableObject { // MARK: - Filter Management - /// Updates filtered entries based on search query + /// Updates filtered entries based on search query, category, and favorite state private func updateFilteredEntries() { - if searchQuery.isEmpty { - let count = min(displayedCount, entries.count) - filteredEntries = Array(entries.prefix(count)) - } else { - let matched = entries.filter { $0.matches(searchQuery) } - let count = min(displayedCount, matched.count) - filteredEntries = Array(matched.prefix(count)) + var result = entries + + // Category filter + if let cat = selectedCategory { + result = result.filter { $0.category == cat } + } + + // Favorites filter + if onlyFavorites { + result = result.filter { $0.isFavorite } + } + + // Search query filter + if !searchQuery.isEmpty { + result = result.filter { $0.matches(searchQuery) } } - hasMoreEntries = filteredEntries.count < entries.count && searchQuery.isEmpty + let count = min(displayedCount, result.count) + filteredEntries = Array(result.prefix(count)) + hasMoreEntries = filteredEntries.count < result.count && searchQuery.isEmpty } // MARK: - Thumbnail Generation diff --git a/TransFrame/Services/ImageExporter+AnnotationRendering.swift b/TransFrame/Services/ImageExporter+AnnotationRendering.swift index 5ce7ac5..66489da 100644 --- a/TransFrame/Services/ImageExporter+AnnotationRendering.swift +++ b/TransFrame/Services/ImageExporter+AnnotationRendering.swift @@ -72,17 +72,92 @@ extension ImageExporter { in context: CGContext, imageHeight: CGFloat ) { - let rect = CGRect( + let cgBubbleRect = CGRect( x: annotation.rect.origin.x, y: imageHeight - annotation.rect.origin.y - annotation.rect.height, width: annotation.rect.width, height: annotation.rect.height ) + let cgTarget = CGPoint( + x: annotation.targetPoint.x, + y: imageHeight - annotation.targetPoint.y + ) + + // Measure the text first; grow the bubble downward (SwiftUI-down = + // CG minY decreasing) when the stored rect is too short for the + // content, so exports never silently truncate what the preview showed. + var bubbleRect = cgBubbleRect + var measuredText: (attributedString: NSAttributedString, framesetter: CTFramesetter, padding: CGFloat)? + if !annotation.text.isEmpty { + let fontSize = max(10, annotation.style.fontSize) + let font = NSFont.systemFont(ofSize: fontSize, weight: .medium) + let attributes: [NSAttributedString.Key: Any] = [ + .font: font, + .foregroundColor: annotation.style.color.nsColor + ] + let attributedString = NSAttributedString(string: annotation.text, attributes: attributes) + let textPadding: CGFloat = 6.0 + let framesetter = CTFramesetterCreateWithAttributedString(attributedString) + measuredText = (attributedString, framesetter, textPadding) + + let insetWidth = max(1, cgBubbleRect.width - textPadding * 2) + let suggested = CTFramesetterSuggestFrameSizeWithConstraints( + framesetter, + CFRangeMake(0, attributedString.length), + nil, + CGSize(width: insetWidth, height: .greatestFiniteMagnitude), + nil + ) + let requiredHeight = ceil(suggested.height) + textPadding * 2 + if requiredHeight > bubbleRect.height { + bubbleRect = CGRect( + x: bubbleRect.minX, + y: bubbleRect.maxY - requiredHeight, + width: bubbleRect.width, + height: requiredHeight + ) + } + } + context.saveGState() + + // 1. Draw pointer tail from bubble to target point + let attachX = max(bubbleRect.minX + 10, min(bubbleRect.maxX - 10, cgTarget.x)) + let attachY = cgTarget.y < bubbleRect.minY ? bubbleRect.minY : (cgTarget.y > bubbleRect.maxY ? bubbleRect.maxY : bubbleRect.midY) + let halfBase = min(8.0, max(4.0, bubbleRect.width * 0.1)) + let tailPath = CGMutablePath() + tailPath.move(to: cgTarget) + tailPath.addLine(to: CGPoint(x: attachX - halfBase, y: attachY)) + tailPath.addLine(to: CGPoint(x: attachX + halfBase, y: attachY)) + tailPath.closeSubpath() context.setFillColor(annotation.bubbleColor.cgColor) - let path = CGPath(roundedRect: rect, cornerWidth: 8, cornerHeight: 8, transform: nil) + context.addPath(tailPath) + context.fillPath() + context.setStrokeColor(annotation.bubbleColor.cgColor) + context.setLineWidth(2.0) + context.move(to: cgTarget) + context.addLine(to: CGPoint(x: attachX, y: attachY)) + context.strokePath() + + // 2. Draw bubble body + context.setFillColor(annotation.bubbleColor.cgColor) + let path = CGPath(roundedRect: bubbleRect, cornerWidth: 8, cornerHeight: 8, transform: nil) context.addPath(path) context.fillPath() + + // 3. Draw text inside bubble + if let measured = measuredText { + let textRect = bubbleRect.insetBy(dx: measured.padding, dy: measured.padding) + let textPath = CGPath(rect: textRect, transform: nil) + let frame = CTFramesetterCreateFrame( + measured.framesetter, + CFRangeMake(0, measured.attributedString.length), + textPath, + nil + ) + CTFrameDraw(frame, context) + } + context.restoreGState() } @@ -101,8 +176,8 @@ extension ImageExporter { ) if annotation.isFilled { - // Filled rectangle - solid color to hide underlying content - context.setFillColor(annotation.style.color.cgColor) + // Filled rectangle - background fill color (falls back to stroke color for legacy docs) + context.setFillColor(annotation.effectiveFillColor.cgColor) context.fill(rect) } else { // Hollow rectangle - outline only @@ -144,6 +219,7 @@ extension ImageExporter { // Transform from SwiftUI coordinates (origin top-left) to CG coordinates (origin bottom-left) let start = CGPoint(x: annotation.startPoint.x, y: imageHeight - annotation.startPoint.y) let end = CGPoint(x: annotation.endPoint.x, y: imageHeight - annotation.endPoint.y) + let control = annotation.controlPoint.map { CGPoint(x: $0.x, y: imageHeight - $0.y) } let lineWidth = annotation.style.lineWidth context.setStrokeColor(annotation.style.color.cgColor) @@ -152,18 +228,23 @@ extension ImageExporter { context.setLineCap(.round) context.setLineJoin(.round) - // Draw the main line + // Draw the main line (curved if control point exists) context.beginPath() context.move(to: start) - context.addLine(to: end) + if let cp = control { + context.addQuadCurve(to: end, control: cp) + } else { + context.addLine(to: end) + } context.strokePath() // Draw the arrowhead let arrowHeadLength = max(lineWidth * 4, 12) let arrowHeadAngle: CGFloat = .pi / 6 - let dx = end.x - start.x - let dy = end.y - start.y + let prevPoint = control ?? start + let dx = end.x - prevPoint.x + let dy = end.y - prevPoint.y let angle = atan2(dy, dx) let arrowPoint1 = CGPoint( @@ -191,32 +272,38 @@ extension ImageExporter { ) { guard !annotation.content.isEmpty else { return } - // Create attributed string + // Create attributed string with word-wrapping so multi-line content is fully rendered let font = NSFont(name: annotation.style.fontName, size: annotation.style.fontSize) ?? NSFont.systemFont(ofSize: annotation.style.fontSize) - + let paragraph = NSMutableParagraphStyle() + paragraph.lineBreakMode = .byWordWrapping let attributes: [NSAttributedString.Key: Any] = [ .font: font, + .paragraphStyle: paragraph, .foregroundColor: annotation.style.color.nsColor ] - let attributedString = NSAttributedString(string: annotation.content, attributes: attributes) - // Draw text at position (transform Y coordinate) - let position = CGPoint( - x: annotation.position.x, - y: imageHeight - annotation.position.y - annotation.style.fontSize + // Frame in the measured bounds so wrapped lines and hard line breaks both fit + let bounds = annotation.bounds + guard bounds.width > 0, bounds.height > 0 else { return } + let frameRect = CGRect( + x: bounds.minX, + y: imageHeight - bounds.maxY, + width: bounds.width, + height: bounds.height ) - // Save context state context.saveGState() - - // Create line and draw - let line = CTLineCreateWithAttributedString(attributedString) - context.textPosition = position - CTLineDraw(line, context) - - // Restore context state + let path = CGPath(rect: frameRect, transform: nil) + let framesetter = CTFramesetterCreateWithAttributedString(attributedString) + let frame = CTFramesetterCreateFrame( + framesetter, + CFRange(location: 0, length: attributedString.length), + path, + nil + ) + CTFrameDraw(frame, context) context.restoreGState() } @@ -234,7 +321,7 @@ extension ImageExporter { ) if annotation.isFilled { - context.setFillColor(annotation.style.color.cgColor) + context.setFillColor(annotation.effectiveFillColor.cgColor) context.fillEllipse(in: rect) } else { context.setStrokeColor(annotation.style.color.cgColor) @@ -396,6 +483,19 @@ extension ImageExporter { ) let radius = annotation.size / 2 + if let lp = annotation.leadPoint { + let leadCG = CGPoint(x: lp.x, y: imageHeight - lp.y) + context.setStrokeColor(annotation.color.cgColor) + context.setLineWidth(2.0) + context.beginPath() + context.move(to: center) + context.addLine(to: leadCG) + context.strokePath() + + context.setFillColor(annotation.color.cgColor) + context.fillEllipse(in: CGRect(x: leadCG.x - 3, y: leadCG.y - 3, width: 6, height: 6)) + } + // Draw filled circle context.setFillColor(annotation.color.cgColor) context.fillEllipse(in: CGRect( diff --git a/TransFrame/Services/ImageExporter.swift b/TransFrame/Services/ImageExporter.swift index e9170fd..bbac8c5 100644 --- a/TransFrame/Services/ImageExporter.swift +++ b/TransFrame/Services/ImageExporter.swift @@ -21,15 +21,22 @@ struct ImageExporter: Sendable { annotations: [Annotation], to url: URL, format: ExportFormat, - quality: Double = 0.9 + quality: Double = 0.9, + beautifyConfig: BeautifyConfig? = nil ) throws { - let finalImage: CGImage + var finalImage: CGImage if annotations.isEmpty { finalImage = image } else { finalImage = try compositeAnnotations(annotations, onto: image) } + if let bConfig = beautifyConfig, bConfig.isEnabled { + if let beautified = BeautifyRenderer.shared.render(image: finalImage, config: bConfig) { + finalImage = beautified + } + } + try writeImage(finalImage, to: url, format: format, quality: quality) } diff --git a/TransFrame/Services/ImageUploadService.swift b/TransFrame/Services/ImageUploadService.swift new file mode 100644 index 0000000..ec38064 --- /dev/null +++ b/TransFrame/Services/ImageUploadService.swift @@ -0,0 +1,468 @@ +import Foundation +import CoreGraphics +import AppKit +import CryptoKit +import Security + +/// Supported image hosting providers. +enum ImageHostType: String, CaseIterable, Codable, Sendable { + case none = "none" + case picgo = "picgo" + case aliyunOSS = "aliyunOSS" + case tencentCOS = "tencentCOS" + case qiniuKodo = "qiniuKodo" + + var displayName: String { + switch self { + case .none: return "None (Disabled)" + case .picgo: return "PicGo (Local App)" + case .aliyunOSS: return "Aliyun OSS" + case .tencentCOS: return "Tencent Cloud COS" + case .qiniuKodo: return "Qiniu Kodo" + } + } +} + +/// Image Host configuration stored locally in AppSettings / UserDefaults. +struct ImageHostSettings: Codable, Sendable, Equatable { + var activeHost: ImageHostType = .none + var picgoEndpoint: String = "http://127.0.0.1:36677/upload" + var copyMarkdownFormat: Bool = true + + // Aliyun OSS + var ossBucket: String = "" + var ossEndpoint: String = "" // e.g. oss-cn-hangzhou.aliyuncs.com + var ossAccessKeyId: String = "" + var ossAccessKeySecret: String = "" + var ossPathPrefix: String = "transframe/" + + // Tencent COS + var cosBucket: String = "" + var cosRegion: String = "" // e.g. ap-guangzhou + var cosSecretId: String = "" + var cosSecretKey: String = "" + var cosPathPrefix: String = "transframe/" + + // Qiniu Kodo + var kodoBucket: String = "" + var kodoDomain: String = "" // e.g. https://cdn.example.com + var kodoAccessKey: String = "" + var kodoSecretKey: String = "" + var kodoPathPrefix: String = "transframe/" + + static let `default` = ImageHostSettings() +} + +extension Notification.Name { + static let imageDidUpload = Notification.Name("imageDidUpload") + static let colorDidPick = Notification.Name("colorDidPick") +} + +/// Upload result representation +struct UploadResult: Sendable { + let url: String + let markdown: String + let provider: ImageHostType +} + +/// Keychain storage for image-host credentials. UserDefaults keeps only the +/// non-sensitive parts of ImageHostSettings — plaintext secrets there are +/// readable by any process on the machine and travel with system backups. +private enum ImageHostSecretStore { + private static let service = "TransFrame.ImageHostSettings" + + private static func baseQuery(account: String) -> [String: Any] { + [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + } + + /// Whether a secret is stored for `account`, checked without fetching + /// the secret data. Callers use this to decide whether to read Keychain + /// at all (e.g. skip the migration read when nothing has been saved yet). + static func exists(account: String) -> Bool { + var query = baseQuery(account: account) + query[kSecReturnAttributes as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitOne + return SecItemCopyMatching(query as CFDictionary, nil) == errSecSuccess + } + + /// Upserts the secret. An empty value deletes the stored entry instead — + /// that is the "user cleared the key in Settings" path, and it must not + /// silently leave the previous secret behind. + static func save(_ value: String, account: String) { + let query = baseQuery(account: account) + guard !value.isEmpty else { + SecItemDelete(query as CFDictionary) + return + } + let data = Data(value.utf8) + let status = SecItemUpdate( + query as CFDictionary, + [kSecValueData as String: data] as CFDictionary + ) + if status == errSecItemNotFound { + var add = query + add[kSecValueData as String] = data + SecItemAdd(add as CFDictionary, nil) + } + } + + static func read(account: String) -> String? { + guard exists(account: account) else { + return nil + } + var query = baseQuery(account: account) + query[kSecReturnData as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitOne + var item: CFTypeRef? + guard SecItemCopyMatching(query as CFDictionary, &item) == errSecSuccess, + let data = item as? Data else { + return nil + } + return String(data: data, encoding: .utf8) + } +} + +/// Service that handles uploading screenshot image data directly to cloud providers or PicGo. +@MainActor +final class ImageUploadService { + static let shared = ImageUploadService() + + private let userDefaultsKey = "TransFrame_ImageHostSettings" + + /// Keychain accounts for the three provider secrets. + private static let secretAccounts = ("ossAccessKeySecret", "cosSecretKey", "kodoSecretKey") + + var settings: ImageHostSettings { + didSet { + saveSettings() + } + } + + private init() { + var loaded: ImageHostSettings + if let data = UserDefaults.standard.data(forKey: userDefaultsKey), + let decoded = try? JSONDecoder().decode(ImageHostSettings.self, from: data) { + loaded = decoded + } else { + loaded = .default + } + + // Secrets live in Keychain; migrate legacy plaintext (from older + // builds that stored the whole struct in UserDefaults) exactly once. + // Only touch Keychain when an entry is actually stored — a brand-new + // install never queries secret data at all. + let (ossAccount, cosAccount, kodoAccount) = Self.secretAccounts + + if ImageHostSecretStore.exists(account: ossAccount), let stored = ImageHostSecretStore.read(account: ossAccount), !stored.isEmpty { + loaded.ossAccessKeySecret = stored + } else if !loaded.ossAccessKeySecret.isEmpty { + ImageHostSecretStore.save(loaded.ossAccessKeySecret, account: ossAccount) + } + if ImageHostSecretStore.exists(account: cosAccount), let stored = ImageHostSecretStore.read(account: cosAccount), !stored.isEmpty { + loaded.cosSecretKey = stored + } else if !loaded.cosSecretKey.isEmpty { + ImageHostSecretStore.save(loaded.cosSecretKey, account: cosAccount) + } + if ImageHostSecretStore.exists(account: kodoAccount), let stored = ImageHostSecretStore.read(account: kodoAccount), !stored.isEmpty { + loaded.kodoSecretKey = stored + } else if !loaded.kodoSecretKey.isEmpty { + ImageHostSecretStore.save(loaded.kodoSecretKey, account: kodoAccount) + } + + self.settings = loaded + // Persist a sanitized JSON (secrets blanked) — also clears any legacy + // plaintext that older builds left in UserDefaults. Idempotent. + saveSettings() + } + + private func saveSettings() { + let (ossAccount, cosAccount, kodoAccount) = Self.secretAccounts + ImageHostSecretStore.save(settings.ossAccessKeySecret, account: ossAccount) + ImageHostSecretStore.save(settings.cosSecretKey, account: cosAccount) + ImageHostSecretStore.save(settings.kodoSecretKey, account: kodoAccount) + + var sanitized = settings + sanitized.ossAccessKeySecret = "" + sanitized.cosSecretKey = "" + sanitized.kodoSecretKey = "" + if let encoded = try? JSONEncoder().encode(sanitized) { + UserDefaults.standard.set(encoded, forKey: userDefaultsKey) + } + } + + var isConfigured: Bool { + settings.activeHost != .none + } + + /// The account name of the secret the active host signs with, or nil + /// when the host needs no key (none / local PicGo). + private var activeHostSecretAccount: String? { + let (ossAccount, cosAccount, kodoAccount) = Self.secretAccounts + switch settings.activeHost { + case .none, .picgo: + return nil + case .aliyunOSS: + return ossAccount + case .tencentCOS: + return cosAccount + case .qiniuKodo: + return kodoAccount + } + } + + /// True when the active host can actually sign an upload — i.e. its + /// secret is stored in Keychain (Settings writes it there on save). + /// Checking before the request avoids a late signature failure from the + /// remote API with no hint that the key is simply missing. + var hasActiveHostSecret: Bool { + guard let account = activeHostSecretAccount else { + return true + } + return ImageHostSecretStore.exists(account: account) + } + + /// Uploads a CGImage using the currently active image host. + func upload(image: CGImage) async throws -> UploadResult { + guard settings.activeHost != .none else { + throw TransFrameError.imageUploadFailed(reason: "no image host configured") + } + guard hasActiveHostSecret else { + throw TransFrameError.imageUploadFailed( + reason: "missing API key for \(settings.activeHost.displayName); save one in Settings first" + ) + } + + // Convert CGImage to PNG Data + let rep = NSBitmapImageRep(cgImage: image) + guard let data = rep.representation(using: .png, properties: [:]) else { + throw TransFrameError.exportEncodingFailed(format: .png) + } + + let filename = "TF_\(Int(Date().timeIntervalSince1970))_\(UUID().uuidString.prefix(6)).png" + let publicURLString: String + + switch settings.activeHost { + case .none: + throw TransFrameError.imageUploadFailed(reason: "no image host configured") + + case .picgo: + publicURLString = try await uploadToPicGo(data: data, filename: filename) + + case .aliyunOSS: + publicURLString = try await uploadToAliyunOSS(data: data, filename: filename) + + case .tencentCOS: + publicURLString = try await uploadToTencentCOS(data: data, filename: filename) + + case .qiniuKodo: + publicURLString = try await uploadToQiniuKodo(data: data, filename: filename) + } + + let markdown = "![\(filename)](\(publicURLString))" + let copiedText = settings.copyMarkdownFormat ? markdown : publicURLString + + // Copy to system clipboard + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(copiedText, forType: .string) + + let result = UploadResult(url: publicURLString, markdown: markdown, provider: settings.activeHost) + NotificationCenter.default.post(name: .imageDidUpload, object: result) + + return result + } + + // MARK: - PicGo Local Upload + + private func uploadToPicGo(data: Data, filename: String) async throws -> String { + guard let endpoint = URL(string: settings.picgoEndpoint) else { + throw TransFrameError.imageUploadFailed(reason: "invalid PicGo endpoint") + } + + let tempFile = FileManager.default.temporaryDirectory.appendingPathComponent(filename) + try data.write(to: tempFile) + defer { try? FileManager.default.removeItem(at: tempFile) } + + var request = URLRequest(url: endpoint) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let payload: [String: Any] = ["list": [tempFile.path]] + request.httpBody = try JSONSerialization.data(withJSONObject: payload) + + let (respData, response) = try await URLSession.shared.data(for: request) + if let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) { + if let json = try? JSONSerialization.jsonObject(with: respData) as? [String: Any], + let resultList = json["result"] as? [String], let firstURL = resultList.first { + return firstURL + } + throw TransFrameError.imageUploadFailed(reason: "unexpected PicGo response payload") + } + let code = (response as? HTTPURLResponse)?.statusCode ?? -1 + throw TransFrameError.imageUploadFailed(reason: "PicGo HTTP \(code)") + } + + // MARK: - Aliyun OSS Upload (Direct REST PUT) + + private func uploadToAliyunOSS(data: Data, filename: String) async throws -> String { + let bucket = settings.ossBucket + let endpoint = settings.ossEndpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + let keyId = settings.ossAccessKeyId + let keySecret = settings.ossAccessKeySecret + let keyPath = settings.ossPathPrefix.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + "/" + filename + + let host = "\(bucket).\(endpoint)" + guard let url = URL(string: "https://\(host)/\(keyPath)") else { + throw TransFrameError.imageUploadFailed(reason: "invalid OSS URL") + } + + let rfc822Formatter = DateFormatter() + rfc822Formatter.locale = Locale(identifier: "en_US_POSIX") + rfc822Formatter.timeZone = TimeZone(secondsFromGMT: 0) + rfc822Formatter.dateFormat = "EEE, dd MMM yyyy HH:mm:ss 'GMT'" + let dateString = rfc822Formatter.string(from: Date()) + + let contentType = "image/png" + let canonicalizedResource = "/\(bucket)/\(keyPath)" + let stringToSign = "PUT\n\n\(contentType)\n\(dateString)\n\(canonicalizedResource)" + + // HMAC-SHA1 + let key = SymmetricKey(data: Data(keySecret.utf8)) + let signatureData = HMAC.authenticationCode(for: Data(stringToSign.utf8), using: key) + let signature = Data(signatureData).base64EncodedString() + let authHeader = "OSS \(keyId):\(signature)" + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue(host, forHTTPHeaderField: "Host") + request.setValue(dateString, forHTTPHeaderField: "Date") + request.setValue(contentType, forHTTPHeaderField: "Content-Type") + request.setValue(authHeader, forHTTPHeaderField: "Authorization") + request.httpBody = data + + let (_, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else { + let code = (response as? HTTPURLResponse)?.statusCode ?? -1 + throw TransFrameError.imageUploadFailed(reason: "OSS HTTP \(code)") + } + + return "https://\(host)/\(keyPath)" + } + + // MARK: - Tencent Cloud COS Upload (Direct REST PUT) + + private func uploadToTencentCOS(data: Data, filename: String) async throws -> String { + let bucket = settings.cosBucket + let region = settings.cosRegion + let secretId = settings.cosSecretId + let secretKey = settings.cosSecretKey + let keyPath = settings.cosPathPrefix.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + "/" + filename + + let host = "\(bucket).cos.\(region).myqcloud.com" + guard let url = URL(string: "https://\(host)/\(keyPath)") else { + throw TransFrameError.imageUploadFailed(reason: "invalid COS URL") + } + + let now = Int(Date().timeIntervalSince1970) + let expire = now + 3600 + let qKeyTime = "\(now);\(expire)" + + // Q-Sign-Algorithm: sha1 + let signKey = HMAC.authenticationCode(for: Data(qKeyTime.utf8), using: SymmetricKey(data: Data(secretKey.utf8))) + let signKeyHex = signKey.map { String(format: "%02x", $0) }.joined() + + let httpMethod = "put" + let uriPath = "/\(keyPath)" + let formatString = "\(httpMethod)\n\(uriPath)\n\nhost=\(host)\n" + let stringToSign = "sha1\n\(qKeyTime)\n\(Insecure.SHA1.hash(data: Data(formatString.utf8)).map { String(format: "%02x", $0) }.joined())\n" + + let signature = HMAC.authenticationCode(for: Data(stringToSign.utf8), using: SymmetricKey(data: Data(signKeyHex.utf8))) + let signatureHex = signature.map { String(format: "%02x", $0) }.joined() + + let authHeader = "q-sign-algorithm=sha1&q-ak=\(secretId)&q-sign-time=\(qKeyTime)&q-key-time=\(qKeyTime)&q-header-list=host&q-url-param-list=&q-signature=\(signatureHex)" + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue(host, forHTTPHeaderField: "Host") + request.setValue(authHeader, forHTTPHeaderField: "Authorization") + request.setValue("image/png", forHTTPHeaderField: "Content-Type") + request.httpBody = data + + let (_, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else { + let code = (response as? HTTPURLResponse)?.statusCode ?? -1 + throw TransFrameError.imageUploadFailed(reason: "COS HTTP \(code)") + } + + return "https://\(host)/\(keyPath)" + } + + // MARK: - Qiniu Kodo Upload (POST Form) + + private func uploadToQiniuKodo(data: Data, filename: String) async throws -> String { + let bucket = settings.kodoBucket + let domain = settings.kodoDomain.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + let accessKey = settings.kodoAccessKey + let secretKey = settings.kodoSecretKey + let keyPath = settings.kodoPathPrefix.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + "/" + filename + + // Generate Qiniu Upload Token (scope = bucket:key, deadline = now + 3600) + let deadline = Int(Date().timeIntervalSince1970) + 3600 + let putPolicy: [String: Any] = ["scope": "\(bucket):\(keyPath)", "deadline": deadline] + let policyData = try JSONSerialization.data(withJSONObject: putPolicy) + let encodedPolicy = policyData.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + + let sign = HMAC.authenticationCode(for: Data(encodedPolicy.utf8), using: SymmetricKey(data: Data(secretKey.utf8))) + let encodedSign = Data(sign).base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + + let uploadToken = "\(accessKey):\(encodedSign):\(encodedPolicy)" + + // POST multipart form to https://upload.qiniup.com + guard let uploadURL = URL(string: "https://upload.qiniup.com") else { + throw TransFrameError.imageUploadFailed(reason: "invalid Kodo upload URL") + } + + let boundary = "Boundary-\(UUID().uuidString)" + var body = Data() + + // token field + body.append("--\(boundary)\r\n".data(using: .utf8)!) + body.append("Content-Disposition: form-data; name=\"token\"\r\n\r\n".data(using: .utf8)!) + body.append("\(uploadToken)\r\n".data(using: .utf8)!) + + // key field + body.append("--\(boundary)\r\n".data(using: .utf8)!) + body.append("Content-Disposition: form-data; name=\"key\"\r\n\r\n".data(using: .utf8)!) + body.append("\(keyPath)\r\n".data(using: .utf8)!) + + // file field + body.append("--\(boundary)\r\n".data(using: .utf8)!) + body.append("Content-Disposition: form-data; name=\"file\"; filename=\"\(filename)\"\r\n".data(using: .utf8)!) + body.append("Content-Type: image/png\r\n\r\n".data(using: .utf8)!) + body.append(data) + body.append("\r\n".data(using: .utf8)!) + + body.append("--\(boundary)--\r\n".data(using: .utf8)!) + + var request = URLRequest(url: uploadURL) + request.httpMethod = "POST" + request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") + request.httpBody = body + + let (_, response) = try await URLSession.shared.data(for: request) + guard let httpResponse = response as? HTTPURLResponse, (200...299).contains(httpResponse.statusCode) else { + let code = (response as? HTTPURLResponse)?.statusCode ?? -1 + throw TransFrameError.imageUploadFailed(reason: "Kodo HTTP \(code)") + } + + let baseURL = domain.hasPrefix("http") ? domain : "https://\(domain)" + return "\(baseURL)/\(keyPath)" + } +} diff --git a/TransFrame/Services/OCREngine.swift b/TransFrame/Services/OCREngine.swift index 1e99400..3c29d7d 100644 --- a/TransFrame/Services/OCREngine.swift +++ b/TransFrame/Services/OCREngine.swift @@ -21,12 +21,6 @@ actor OCREngine { /// Shared instance for app-wide OCR operations static let shared = OCREngine() - /// Supported recognition languages - private var supportedLanguages: Set = [] - - /// Whether an OCR operation is currently in progress - private var isProcessing = false - // MARK: - Recognition Language /// Text recognition languages supported by Vision framework @@ -52,6 +46,31 @@ actor OCREngine { rawValue } + /// Maps a UI source-language preference to a Vision recognition + /// language. Returns nil for `.auto` and languages Vision does not + /// support, letting the caller fall back to automatic detection. + init?(translationLanguage language: TranslationLanguage) { + switch language { + case .auto: return nil + case .english: self = .english + case .chineseSimplified: self = .chineseSimplified + case .chineseTraditional: self = .chineseTraditional + case .japanese: self = .japanese + case .korean: self = .korean + case .french: self = .french + case .german: self = .german + case .spanish: self = .spanish + case .italian: self = .italian + case .portuguese: self = .portuguese + case .russian: self = .russian + case .arabic: self = .arabic + case .hindi: self = .hindi + case .thai: self = .thai + case .vietnamese: self = .vietnamese + default: return nil + } + } + /// Localized display name var localizedName: String { switch self { @@ -131,13 +150,6 @@ actor OCREngine { _ image: CGImage, config: Configuration = .default ) async throws -> OCRResult { - // Prevent concurrent OCR operations - guard !isProcessing else { - throw OCREngineError.operationInProgress - } - isProcessing = true - defer { isProcessing = false } - // Validate image guard image.width > 0 && image.height > 0 else { throw OCREngineError.invalidImage @@ -152,13 +164,32 @@ actor OCREngine { os_signpost(.begin, log: Self.performanceLog, name: "OCRRecognition", signpostID: Self.signpostID) let startTime = CFAbsoluteTimeGetCurrent() - let handler = VNImageRequestHandler(cgImage: image, options: [:]) - + // VNImageRequestHandler.perform is synchronous and CPU-bound. Run it + // on a global queue so it doesn't block this actor's executor, and so + // concurrent recognize() callers don't serialize behind each other. + // Each call builds its own handler and request; nothing is shared + // across threads, so the unsafe wrapper only silences Sendable noise. + let boxedImage = UnsafeSendable(value: image) + let boxedRequest = UnsafeSendable(value: request) + let texts: [OCRText] do { - try handler.perform([request]) + texts = try await withCheckedThrowingContinuation { continuation in + DispatchQueue.global(qos: .userInitiated).async { + let handler = VNImageRequestHandler(cgImage: boxedImage.value, options: [:]) + do { + try handler.perform([boxedRequest.value]) + let observations = boxedRequest.value.results ?? [] + continuation.resume(returning: observations.compactMap { + OCRText.from($0, imageSize: imageSize) + }) + } catch { + continuation.resume(throwing: OCREngineError.recognitionFailed(underlying: error)) + } + } + } } catch { os_signpost(.end, log: Self.performanceLog, name: "OCRRecognition", signpostID: Self.signpostID) - throw OCREngineError.recognitionFailed(underlying: error) + throw error } let duration = (CFAbsoluteTimeGetCurrent() - startTime) * 1000 @@ -168,16 +199,6 @@ actor OCREngine { os_log("OCR recognition completed in %.1fms", log: OSLog.default, type: .info, duration) #endif - // Extract results - guard let observations = request.results else { - return OCRResult.empty(imageSize: imageSize) - } - - // Convert to OCRText - let texts = observations.compactMap { obs in - OCRText.from(obs, imageSize: imageSize) - } - // Filter by confidence let filteredTexts = texts.filter { $0.confidence >= config.minimumConfidence } @@ -211,54 +232,6 @@ actor OCREngine { return try await recognize(image, config: config) } - // MARK: - Language Detection - - /// Detects the primary language in an image. - /// - Parameter image: The image to analyze - /// - Returns: Detected language, or nil if detection failed - func detectLanguage(in image: CGImage) async -> RecognitionLanguage? { - // Try each language and return the one with the best results - let languagesToTest: [RecognitionLanguage] = [ - .english, - .chineseSimplified, - .chineseTraditional, - .japanese, - .korean - ] - - var bestLanguage: RecognitionLanguage? - var bestConfidence: Float = 0.0 - - for language in languagesToTest { - do { - var config = Configuration.default - config.languages = [language] - config.useAutoLanguageDetection = false - config.recognitionLevel = .fast - config.prefersFastRecognition = true - config.minimumConfidence = 0.3 - - let result = try await recognize(image, config: config) - - if result.hasResults { - let avgConfidence = result.observations - .map(\.confidence) - .reduce(0, +) / Float(result.observations.count) - - if avgConfidence > bestConfidence { - bestConfidence = avgConfidence - bestLanguage = language - } - } - } catch { - // Try next language - continue - } - } - - return bestLanguage - } - // MARK: - Private Methods /// Creates a configured VNRecognizeTextRequest @@ -268,15 +241,11 @@ actor OCREngine { // Set recognition level request.recognitionLevel = config.recognitionLevel.visionLevel - // Enable automatic language detection if requested - if config.useAutoLanguageDetection { - request.usesLanguageCorrection = true - } else { - // Set specific languages + // With auto detection, leave recognitionLanguages at Vision's default + if !config.useAutoLanguageDetection { request.recognitionLanguages = Array(config.languages).map(\.visionLanguage) } - // Enable text recognition for non-horizontal text request.usesLanguageCorrection = true return request @@ -287,9 +256,6 @@ actor OCREngine { /// Errors that can occur during OCR operations enum OCREngineError: LocalizedError, Sendable { - /// OCR operation is already in progress - case operationInProgress - /// The provided image is invalid or empty case invalidImage @@ -304,8 +270,6 @@ enum OCREngineError: LocalizedError, Sendable { var errorDescription: String? { switch self { - case .operationInProgress: - return NSLocalizedString("error.ocr.in.progress", comment: "") case .invalidImage: return NSLocalizedString("error.ocr.invalid.image", comment: "") case .recognitionFailed: @@ -319,8 +283,6 @@ enum OCREngineError: LocalizedError, Sendable { var recoverySuggestion: String? { switch self { - case .operationInProgress: - return NSLocalizedString("error.ocr.in.progress.recovery", comment: "") case .invalidImage: return NSLocalizedString("error.ocr.invalid.image.recovery", comment: "") case .recognitionFailed: @@ -332,3 +294,10 @@ enum OCREngineError: LocalizedError, Sendable { } } } + +/// Wraps a value so it can cross an isolation boundary. Safe here because the +/// wrapped value is created and used exclusively inside a single dispatch +/// closure (`OCREngine.recognize`), never shared between threads. +private struct UnsafeSendable: @unchecked Sendable { + let value: T +} diff --git a/TransFrame/Services/OllamaVLMProvider.swift b/TransFrame/Services/OllamaVLMProvider.swift index f4a9bb4..6395955 100644 --- a/TransFrame/Services/OllamaVLMProvider.swift +++ b/TransFrame/Services/OllamaVLMProvider.swift @@ -66,14 +66,24 @@ struct OllamaVLMProvider: VLMProvider, Sendable { } func analyze(image: CGImage) async throws -> ScreenAnalysisResult { - guard let imageData = image.jpegData(quality: 0.85), !imageData.isEmpty else { - throw VLMProviderError.imageEncodingFailed - } + try await analyze(image: image, prompt: .ocr) + } - let base64Image = imageData.base64EncodedString() + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult { + let encoded = try AnalysisImageEncoder.jpeg(image) let imageSize = CGSize(width: image.width, height: image.height) - let request = try buildRequest(base64Image: base64Image) - let responseData = try await executeRequest(request) + + var responseData: Data + do { + let request = try buildRequest(base64Image: encoded.base64String, prompts: prompt, includeResponseFormat: true) + responseData = try await executeRequest(request) + } catch VLMProviderError.invalidConfiguration { + // Some OpenAI-compatible endpoints reject response_format with a + // 400; retry once without it (mirrors OpenAIVLMProvider). + let request = try buildRequest(base64Image: encoded.base64String, prompts: prompt, includeResponseFormat: false) + responseData = try await executeRequest(request) + } + let vlmResponse = try parseOllamaResponse(responseData) return vlmResponse.toScreenAnalysisResult(imageSize: imageSize) @@ -125,7 +135,7 @@ struct OllamaVLMProvider: VLMProvider, Sendable { } /// Builds the URLRequest for local server (Ollama /api/generate or OpenAI /chat/completions) - private func buildRequest(base64Image: String) throws -> URLRequest { + private func buildRequest(base64Image: String, prompts: VLMPromptPair = .ocr, includeResponseFormat: Bool = true) throws -> URLRequest { let isChatCompletions = configuration.baseURL.path.contains("v1") || configuration.baseURL.path.contains("chat/completions") let endpoint: URL if isChatCompletions { @@ -149,31 +159,37 @@ struct OllamaVLMProvider: VLMProvider, Sendable { request.timeoutInterval = timeout if isChatCompletions { - let body: [String: Any] = [ + var body: [String: Any] = [ "model": configuration.modelName, "messages": [ [ "role": "user", "content": [ - ["type": "text", "text": "\(VLMPromptTemplate.systemPrompt)\n\n\(VLMPromptTemplate.userPrompt)"], + ["type": "text", "text": "\(prompts.system)\n\n\(prompts.user)"], ["type": "image_url", "image_url": ["url": "data:image/jpeg;base64,\(base64Image)"]] ] ] ], "temperature": 0.1 ] + if includeResponseFormat { + // Ollama's OpenAI-compatible layer and most vLLM builds accept + // this; incompatible servers hit the 400 fallback in analyze. + body["response_format"] = ["type": "json_object"] + } request.httpBody = try JSONSerialization.data(withJSONObject: body) } else { let prompt = """ - \(VLMPromptTemplate.systemPrompt) + \(prompts.system) - \(VLMPromptTemplate.userPrompt) + \(prompts.user) """ let requestBody = OllamaGenerateRequest( model: configuration.modelName, prompt: prompt, images: [base64Image], + format: "json", stream: false, options: OllamaOptions( temperature: 0.1, @@ -315,6 +331,9 @@ private struct OllamaGenerateRequest: Encodable, Sendable { let model: String let prompt: String let images: [String] + /// "json" restricts generation to valid JSON, preventing prose responses + /// that would fall through to the plain-text bbox fallback. + let format: String? let stream: Bool let options: OllamaOptions? } diff --git a/TransFrame/Services/OpenAIVLMProvider.swift b/TransFrame/Services/OpenAIVLMProvider.swift index faf52bd..2e5fda0 100644 --- a/TransFrame/Services/OpenAIVLMProvider.swift +++ b/TransFrame/Services/OpenAIVLMProvider.swift @@ -30,6 +30,8 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { let apiMode: OpenAIAPIMode private let logger = Logger.translation + /// Logger for the stateless response parser, usable without an instance. + private static let parserLogger = Logger.translation // MARK: - Initialization @@ -93,20 +95,21 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { private let maxContinuationAttempts = 3 func analyze(image: CGImage) async throws -> ScreenAnalysisResult { - guard let imageData = image.jpegData(quality: 0.85), !imageData.isEmpty else { - throw VLMProviderError.imageEncodingFailed - } + try await analyze(image: image, prompt: .ocr) + } - let base64Image = imageData.base64EncodedString() + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult { + let encoded = try AnalysisImageEncoder.jpeg(image) let imageSize = CGSize(width: image.width, height: image.height) - - logDebug("Image size: \(image.width)x\(image.height), JPEG bytes: \(imageData.count), base64 chars: \(base64Image.count)") + + logDebug("Image size: \(image.width)x\(image.height), \(encoded.mimeType) bytes: \(encoded.data.count), base64 chars: \(encoded.base64String.count)") // Use multi-turn conversation with continuation support let vlmResponse = try await analyzeWithContinuation( - base64Image: base64Image, + base64Image: encoded.base64String, imageSize: imageSize, - maxAttempts: maxContinuationAttempts + maxAttempts: maxContinuationAttempts, + prompts: prompt ) return vlmResponse.toScreenAnalysisResult(imageSize: imageSize) @@ -137,19 +140,21 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { private func analyzeWithContinuation( base64Image: String, imageSize: CGSize, - maxAttempts: Int + maxAttempts: Int, + prompts: VLMPromptPair = .ocr ) async throws -> VLMAnalysisResponse { var allSegments: [VLMTextSegment] = [] + var structuredOutputEnabled = true var conversationMessages: [OpenAIChatMessage] = [ OpenAIChatMessage( role: "system", - content: .text(VLMPromptTemplate.systemPrompt) + content: .text(prompts.system) ), OpenAIChatMessage( role: "user", content: .vision([ - .text(VLMPromptTemplate.userPrompt), + .text(prompts.user), .imageURL(OpenAIImageURL( url: "data:image/jpeg;base64,\(base64Image)" )), @@ -159,10 +164,28 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { for attempt in 0.. 0 - let request = try buildRequest(messages: conversationMessages, isContinuation: isContinuation) - let responseData = try await executeRequest(request) + var responseData: Data + do { + let request = try buildRequest( + messages: conversationMessages, + isContinuation: isContinuation, + includeResponseFormat: structuredOutputEnabled + ) + responseData = try await executeRequest(request) + } catch VLMProviderError.invalidConfiguration where structuredOutputEnabled { + // Some OpenAI-compatible endpoints reject response_format + // (400). Retry the same attempt once without it. + logWarning("Endpoint rejected response_format, retrying without structured output") + structuredOutputEnabled = false + let request = try buildRequest( + messages: conversationMessages, + isContinuation: isContinuation, + includeResponseFormat: false + ) + responseData = try await executeRequest(request) + } - let (content, isTruncated, finishReason) = try extractContentAndStatus(from: responseData) + let (content, isTruncated, finishReason) = try Self.extractContentAndStatus(from: responseData) logDebug("Attempt \(attempt + 1)/\(maxAttempts): received \(content.count) chars, finish reason: \(finishReason ?? "unknown")") @@ -251,9 +274,11 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { } } - /// Extracts content text and truncation status from OpenAI response - func extractContentAndStatus(from data: Data) throws -> (content: String, isTruncated: Bool, finishReason: String?) { - logDebug("Received raw response payload: \(data.count) bytes") + /// Extracts content text and truncation status from an OpenAI response. + /// Stateless — declared static so connection probes can validate a 200 + /// body without constructing a fully configured provider instance. + static func extractContentAndStatus(from data: Data) throws -> (content: String, isTruncated: Bool, finishReason: String?) { + Self.parserLogger.debug("Received raw response payload: \(data.count, privacy: .public)") // Check for error response first if let errorResponse = try? JSONDecoder().decode(OpenAIErrorResponse.self, from: data), @@ -261,17 +286,43 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { throw VLMProviderError.invalidResponse(errorResponse.error.message) } - // Check if response matches Responses API format - if let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { - if let outputText = json["output_text"] as? String, !outputText.isEmpty { - return (outputText, false, nil) + // Responses API 的 reasoning 可以排在 message 之前,不能只读 output[0]。 + if let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + json["object"] as? String == "response" || json["output"] is [[String: Any]] || json["output_text"] is String { + let status = json["status"] as? String + let details = json["incomplete_details"] as? [String: Any] + let reason = details?["reason"] as? String + // JSON null 会解码成 NSNull;字段存在不代表生成被截断。 + let isTruncated = status == "incomplete" && (reason == nil || reason == "max_output_tokens") + if let status, ["failed", "cancelled", "queued", "in_progress"].contains(status) { + throw VLMProviderError.invalidResponse("Responses API status: \(status)") + } + if status == "incomplete", !isTruncated { + throw VLMProviderError.invalidResponse("Responses API incomplete: \(reason ?? "unknown")") } - if let output = json["output"] as? [[String: Any]], - let first = output.first, - let content = first["content"] as? [[String: Any]], - let text = content.first?["text"] as? String { - return (text, false, nil) + if let outputText = json["output_text"] as? String, + !outputText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return (outputText, isTruncated, isTruncated ? "length" : nil) } + let output = json["output"] as? [[String: Any]] ?? [] + let texts = output.filter { item in + item["type"] as? String == "message" || item["type"] == nil + }.flatMap { item in + (item["content"] as? [[String: Any]] ?? []).compactMap { block -> String? in + let type = block["type"] as? String + guard type == nil || type == "output_text" || type == "text", + let text = block["text"] as? String, + !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + return text + } + } + guard !texts.isEmpty else { + let types = output.compactMap { $0["type"] as? String }.joined(separator: ", ") + throw VLMProviderError.invalidResponse( + "Responses API returned no message text (status: \(status ?? "unknown"), output types: \(types))" + ) + } + return (texts.joined(separator: "\n"), isTruncated, isTruncated ? "length" : nil) } // Try to parse as OpenAI response @@ -281,16 +332,6 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { do { openAIResponse = try decoder.decode(OpenAIChatResponse.self, from: data) } catch { - // If JSON decoding fails, try to extract content manually using regex - // This handles cases where the JSON structure is broken - if let rawJSON = String(data: data, encoding: .utf8), - let content = extractContentManually(from: rawJSON) { - let isTruncated = rawJSON.contains("\"finish_reason\":\"length\"") || - rawJSON.contains("\"finish_reason\": \"length\"") - logDebug("Manually extracted content (truncated: \(isTruncated))") - return (content, isTruncated, isTruncated ? "length" : nil) - } - throw VLMProviderError.parsingFailed( "Failed to decode OpenAI response: \(error.localizedDescription). Response length: \(data.count) bytes" ) @@ -311,10 +352,10 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { // Fallback to reasoningContent or reasoning if content is empty if content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { if let reasoning = message.reasoningContent, !reasoning.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - logInfo("Main content is empty, falling back to reasoning_content") + Self.parserLogger.info("Main content is empty, falling back to reasoning_content") content = reasoning } else if let reasoningObj = message.reasoning, !reasoningObj.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - logInfo("Main content is empty, falling back to reasoning") + Self.parserLogger.info("Main content is empty, falling back to reasoning") content = reasoningObj } } @@ -401,8 +442,17 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { // MARK: - Private Methods + /// Placeholder replacing the base64 image in continuation rounds, so the + /// same capture is not uploaded again for every truncation retry. + private static let continuationImagePlaceholder = + "(image omitted: the same image as in the first message)" + /// Builds the URLRequest for OpenAI Chat Completions or Responses API with custom messages - private func buildRequest(messages: [OpenAIChatMessage], isContinuation: Bool = false) throws -> URLRequest { + private func buildRequest( + messages: [OpenAIChatMessage], + isContinuation: Bool = false, + includeResponseFormat: Bool = true + ) throws -> URLRequest { let endpoint: URL if apiMode == .response { endpoint = configuration.baseURL.resolvingLocalhost.appendingPathComponent("responses") @@ -419,9 +469,29 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { } request.timeoutInterval = timeout + // Continuation rounds keep the conversation shape but drop only the + // base64 image part — the text instruction stays so the model still + // knows the task (mirrors ClaudeVLMProvider's per-block replacement). + let effectiveMessages: [OpenAIChatMessage] + if isContinuation { + effectiveMessages = messages.map { message in + guard case .vision(let parts) = message.content else { return message } + var kept: [OpenAIChatMessage.VisionContent] = parts.compactMap { part in + if case .text(let text) = part { return .text(text) } + return nil + } + if kept.isEmpty { + kept = [.text(Self.continuationImagePlaceholder)] + } + return OpenAIChatMessage(role: message.role, content: .vision(kept)) + } + } else { + effectiveMessages = messages + } + // Responses API if apiMode == .response { - let inputItems: [[String: Any]] = messages.map { msg in + let inputItems: [[String: Any]] = effectiveMessages.map { msg in switch msg.content { case .text(let t): return ["role": msg.role, "content": t] @@ -437,28 +507,33 @@ struct OpenAIVLMProvider: VLMProvider, Sendable { return ["role": msg.role, "content": contentArray] } } - let requestBody: [String: Any] = [ + var requestBody: [String: Any] = [ "model": configuration.modelName, "input": inputItems, "temperature": 0.1 ] + if includeResponseFormat { + requestBody["text"] = ["format": ["type": "json_object"]] + } request.httpBody = try JSONSerialization.data(withJSONObject: requestBody) return request } - // Use lower max_tokens for local models (they're slower and don't need as many) + // Local models are slower; 4096 covers ~60-80 segments before + // continuation kicks in, which is cheaper than a second full request. let maxTokens: Int if isLocalEndpoint { - maxTokens = isContinuation ? 2048 : 1024 + maxTokens = 4096 } else { maxTokens = isContinuation ? 16384 : 8192 } let requestBody = OpenAIChatRequest( model: configuration.modelName, - messages: messages, + messages: effectiveMessages, maxTokens: maxTokens, - temperature: 0.1 + temperature: 0.1, + responseFormat: includeResponseFormat ? .jsonObject : nil ) let encoder = JSONEncoder() @@ -728,16 +803,25 @@ private struct OpenAIChatRequest: Encodable, Sendable { let messages: [OpenAIChatMessage] let maxTokens: Int let temperature: Double + let responseFormat: OpenAIResponseFormat? let stream: Bool = false enum CodingKeys: String, CodingKey { - case model, messages + case model, messages, temperature, stream case maxTokens = "max_tokens" - case temperature - case stream + case responseFormat = "response_format" } } +/// OpenAI `response_format` payload forcing JSON output, so providers that +/// support it never emit prose that would fall through to the plain-text +/// fallback (which fabricates bounding boxes). +private struct OpenAIResponseFormat: Encodable, Sendable { + static let jsonObject = OpenAIResponseFormat(type: "json_object") + + let type: String +} + /// Chat message with support for vision content private struct OpenAIChatMessage: Encodable, Sendable { let role: String diff --git a/TransFrame/Services/PaddleOCREngine.swift b/TransFrame/Services/PaddleOCREngine.swift index 37176f5..2f37d5e 100644 --- a/TransFrame/Services/PaddleOCREngine.swift +++ b/TransFrame/Services/PaddleOCREngine.swift @@ -22,6 +22,53 @@ actor PaddleOCREngine { /// Maximum concurrent operations private var isProcessing = false + /// FIFO queue of callers waiting for the in-flight recognition; local + /// PaddleOCR runs a heavyweight Python process, so calls serialize. + private var executionWaiters: [ExecutionWaiter] = [] + + /// Identifiable waiter so a cancelled task can dequeue itself. + private struct ExecutionWaiter: Sendable { + let id: UUID + let continuation: CheckedContinuation + } + + /// Waits for any in-flight recognition to finish instead of failing + /// with `operationInProgress`. Throws `CancellationError` when the caller + /// is cancelled while still queued, instead of running every queued + /// Python process first. + private func acquireExecutionSlot() async throws { + guard isProcessing else { + isProcessing = true + return + } + let id = UUID() + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + executionWaiters.append(ExecutionWaiter(id: id, continuation: continuation)) + } + } onCancel: { + Task { await self.dequeueAndCancel(id: id) } + } + } + + /// Removes a queued waiter (if still queued) and resumes it with a + /// cancellation error. Running actor isolation makes this race-free with + /// `releaseExecutionSlot`: whichever runs first removes the waiter. + private func dequeueAndCancel(id: UUID) { + guard let index = executionWaiters.firstIndex(where: { $0.id == id }) else { return } + let waiter = executionWaiters.remove(at: index) + waiter.continuation.resume(throwing: CancellationError()) + } + + /// Hands the slot to the next waiter, or frees it when nobody is queued. + private func releaseExecutionSlot() { + if let next = executionWaiters.first { + executionWaiters.removeFirst() + next.continuation.resume() + } else { + isProcessing = false + } + } // MARK: - Configuration @@ -132,21 +179,23 @@ actor PaddleOCREngine { _ image: CGImage, config: Configuration = .default ) async throws -> OCRResult { - // Prevent concurrent operations - guard !isProcessing else { - throw PaddleOCREngineError.operationInProgress - } - isProcessing = true - defer { isProcessing = false } + // Serialize behind any in-flight recognition (queued, not rejected) + try await acquireExecutionSlot() + defer { releaseExecutionSlot() } // Validate image guard image.width > 0 && image.height > 0 else { throw PaddleOCREngineError.invalidImage } + // Downsample Retina captures once; everything downstream (temp file, + // cloud upload, coordinate normalization) works against this image. + let analysisImage = AnalysisImageEncoder.downsampled(image) + let analysisSize = CGSize(width: analysisImage.width, height: analysisImage.height) + // Use cloud API if configured if config.useCloud { - return try await recognizeViaCloudAPI(image: image, config: config) + return try await recognizeViaCloudAPI(image: analysisImage, config: config) } // Local mode - check availability @@ -155,7 +204,7 @@ actor PaddleOCREngine { } // Save image to temporary file - let tempURL = try saveImageToTempFile(image) + let tempURL = try saveImageToTempFile(analysisImage) defer { // Clean up temp file @@ -169,14 +218,14 @@ actor PaddleOCREngine { let result = try await executePaddleOCR(arguments: arguments) // Parse output - let observations = try parsePaddleOCROutput(result, imageSize: CGSize(width: image.width, height: image.height), mode: config.mode) + let observations = try parsePaddleOCROutput(result, imageSize: analysisSize, mode: config.mode) // Filter by confidence let filteredTexts = observations.filter { $0.confidence >= config.minimumConfidence } return OCRResult( observations: filteredTexts, - imageSize: CGSize(width: image.width, height: image.height) + imageSize: analysisSize ) } @@ -232,18 +281,13 @@ actor PaddleOCREngine { } // Convert image to base64 data URL - guard let data = CFDataCreateMutable(nil, 0) else { - throw PaddleOCREngineError.failedToSaveImage - } - guard let destination = CGImageDestinationCreateWithData(data, UTType.png.identifier as CFString, 1, nil) else { - throw PaddleOCREngineError.failedToSaveImage - } - CGImageDestinationAddImage(destination, image, nil) - guard CGImageDestinationFinalize(destination) else { + let encoded: AnalysisImageEncoder.EncodedImage + do { + encoded = try AnalysisImageEncoder.png(image) + } catch { throw PaddleOCREngineError.failedToSaveImage } - let base64Image = (data as Data).base64EncodedString() - let imageURL = "data:image/png;base64,\(base64Image)" + let imageURL = encoded.dataURI // Build OpenAI-compatible request var request = URLRequest(url: apiURL) @@ -927,9 +971,6 @@ enum PaddleOCREngineError: LocalizedError, Sendable { /// PaddleOCR is not installed case notInstalled - /// OCR operation is already in progress - case operationInProgress - /// The provided image is invalid or empty case invalidImage @@ -955,11 +996,6 @@ enum PaddleOCREngineError: LocalizedError, Sendable { "error.paddleocr.not.installed", comment: "PaddleOCR is not installed" ) - case .operationInProgress: - return NSLocalizedString( - "error.ocr.in.progress", - comment: "OCR operation is already in progress" - ) case .invalidImage: return NSLocalizedString( "error.ocr.invalid.image", @@ -994,11 +1030,6 @@ enum PaddleOCREngineError: LocalizedError, Sendable { "error.paddleocr.not.installed.recovery", comment: "Install PaddleOCR using: pip install paddleocr" ) - case .operationInProgress: - return NSLocalizedString( - "error.ocr.in.progress.recovery", - comment: "Wait for the current operation to complete" - ) case .invalidImage: return NSLocalizedString( "error.ocr.invalid.image.recovery", diff --git a/TransFrame/Services/ScrollCaptureSession.swift b/TransFrame/Services/ScrollCaptureSession.swift new file mode 100644 index 0000000..e437303 --- /dev/null +++ b/TransFrame/Services/ScrollCaptureSession.swift @@ -0,0 +1,202 @@ +import AppKit +import CoreGraphics +import Vision +import os +@preconcurrency import ScreenCaptureKit + +/// Manages a continuous scroll screenshot session, performing hardware-accelerated image registration +/// and vertical stitching using ScreenCaptureKit and Vision framework. +@MainActor +final class ScrollCaptureSession { + private static let logger = Logger(subsystem: "com.transframe.app", category: "ScrollCapture") + + /// Hard cap on the stitched bitmap height; stops the session instead of + /// growing memory without bound during long scrolls (10 FPS loop). + private static let maxStitchHeight = 20_000 + + private let displayID: CGDirectDisplayID + private let screenRect: CGRect // in global quartz coordinates + private var isCapturing: Bool = false + private var isProcessing: Bool = false + private var timer: Timer? + + /// Display lookup cached for the session lifetime — re-enumerating all + /// windows via SCShareableContent every frame is expensive on MainActor. + private var cachedDisplay: SCDisplay? + + private var previousImage: CGImage? + private var stitchedImage: CGImage? + private var totalHeight: CGFloat = 0 + + var onUpdate: ((CGImage) -> Void)? + + init(displayID: CGDirectDisplayID, screenRect: CGRect) { + self.displayID = displayID + self.screenRect = screenRect + } + + /// Starts continuous capture and image registration loop. + func start() { + guard !isCapturing else { return } + isCapturing = true + + Task { @MainActor in + if let initial = await self.captureCurrentSlice() { + self.previousImage = initial + self.stitchedImage = initial + self.totalHeight = CGFloat(initial.height) + self.onUpdate?(initial) + } + } + + // 10 FPS capture loop + timer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in + Task { @MainActor in + await self?.processNextSlice() + } + } + } + + /// Stops continuous capture and returns the final stitched image. + func finish() -> CGImage? { + timer?.invalidate() + timer = nil + isCapturing = false + return stitchedImage + } + + /// Cancels capture and discards stitched buffer. + func cancel() { + timer?.invalidate() + timer = nil + isCapturing = false + stitchedImage = nil + previousImage = nil + } + + // MARK: - Frame Registration & Stitching + + private func processNextSlice() async { + guard isCapturing, !isProcessing else { return } + isProcessing = true + defer { isProcessing = false } + + guard let currentSlice = await captureCurrentSlice(), let prev = previousImage else { return } + + // Perform translation registration using Apple Vision framework. + // CPU-bound work runs off MainActor; only the ty scalar crosses back. + let request = VNTranslationalImageRegistrationRequest(targetedCGImage: currentSlice, options: [:]) + let handler = VNImageRequestHandler(cgImage: prev, options: [:]) + let boxedRequest = RegistrationBox(value: request) + let boxedHandler = RegistrationBox(value: handler) + + let dy: CGFloat? + do { + dy = try await Task.detached(priority: .userInitiated) { () -> CGFloat? in + try boxedHandler.value.perform([boxedRequest.value]) + let observation = boxedRequest.value.results?.first as? VNImageTranslationAlignmentObservation + return observation.map { CGFloat($0.alignmentTransform.ty) } + }.value + } catch { + Self.logger.debug("Image registration failed: \(error.localizedDescription)") + return + } + + guard let dy else { return } + + // alignmentTransform maps the target (prev) onto the requested image + // (currentSlice); the sign convention below ("current moved down") + // still needs validation against a real scroll session — a flipped + // sign silently discards every frame. + if dy > 4.0 && dy < CGFloat(currentSlice.height) * 0.9 { + appendSlice(currentSlice, deltaY: dy) + previousImage = currentSlice + } + } + + private func appendSlice(_ slice: CGImage, deltaY: CGFloat) { + guard let base = stitchedImage else { + stitchedImage = slice + return + } + + let width = base.width + let addedHeight = Int(round(deltaY)) + let newTotalHeight = base.height + addedHeight + + guard newTotalHeight <= Self.maxStitchHeight else { + Self.logger.error("Stitched height limit reached (\(newTotalHeight, privacy: .public)px); stopping scroll capture") + _ = finish() + return + } + + guard let colorSpace = base.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB) else { + _ = finish() + return + } + guard let context = CGContext( + data: nil, + width: width, + height: newTotalHeight, + bitsPerComponent: 8, + bytesPerRow: 0, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + // Silently returning would freeze the stitch on the old frame + // with no signal; end the session so the user keeps what exists. + Self.logger.error("Stitch context allocation failed at \(newTotalHeight, privacy: .public)px; stopping scroll capture") + _ = finish() + return + } + + // Draw existing base image shifted up + context.draw(base, in: CGRect(x: 0, y: addedHeight, width: width, height: base.height)) + + // Draw newly scrolled bottom slice + context.draw(slice, in: CGRect(x: 0, y: 0, width: width, height: slice.height)) + + if let newStitched = context.makeImage() { + self.stitchedImage = newStitched + self.totalHeight = CGFloat(newTotalHeight) + self.onUpdate?(newStitched) + } + } + + private func captureCurrentSlice() async -> CGImage? { + do { + // Reuse the session-scoped display lookup; only fall back to a + // fresh enumeration when the cache is empty. + let scDisplay: SCDisplay + if let cached = cachedDisplay { + scDisplay = cached + } else { + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + guard let resolved = content.displays.first(where: { $0.displayID == self.displayID }) ?? content.displays.first else { + return nil + } + cachedDisplay = resolved + scDisplay = resolved + } + let filter = SCContentFilter(display: scDisplay, excludingWindows: []) + let config = SCStreamConfiguration() + config.sourceRect = screenRect + let scale: CGFloat = NSScreen.main?.backingScaleFactor ?? 2.0 + config.width = max(1, Int(screenRect.width * scale)) + config.height = max(1, Int(screenRect.height * scale)) + config.showsCursor = false + config.scalesToFit = false + return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + } catch { + Self.logger.debug("Scroll slice capture failed: \(error.localizedDescription)") + return nil + } + } +} + +/// Wraps a Vision request/handler so the registration work can cross into a +/// detached task. Safe here: the wrapped values are created and used +/// exclusively inside that one task, never shared. +private struct RegistrationBox: @unchecked Sendable { + let value: T +} diff --git a/TransFrame/Services/SmartPrivacyRedactionService.swift b/TransFrame/Services/SmartPrivacyRedactionService.swift new file mode 100644 index 0000000..b2a88a4 --- /dev/null +++ b/TransFrame/Services/SmartPrivacyRedactionService.swift @@ -0,0 +1,75 @@ +import Foundation +import CoreGraphics +import os + +/// Detects sensitive privacy regions (phone numbers, passwords, amounts, +/// faces, keys, emails, IDs) in an image using the configured online VLM +/// provider, returning rects in image pixel coordinates (top-left origin). +final class SmartPrivacyRedactionService: Sendable { + static let shared = SmartPrivacyRedactionService() + + private let logger = Logger.ui + + enum PrivacyDetectionError: LocalizedError { + case providerNotConfigured + + var errorDescription: String? { + switch self { + case .providerNotConfigured: + return String(localized: "smartMosaic.error.notConfigured") + } + } + } + + private init() {} + + /// Whether the shared VLM provider configuration is ready. + /// Smart mosaic uses the same VLM settings as screen analysis — no separate config. + @MainActor + var isConfigured: Bool { + let settings = AppSettings.shared + let provider = settings.vlmProvider + // Local providers (Ollama etc.) need no API key + if !provider.requiresAPIKey { return true } + // In-memory key (may be freshly typed in Settings) … + if !settings.vlmAPIKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return true + } + // … or the persisted per-provider key in Keychain + return !settings.vlmAPIKey(for: provider).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + /// Detects privacy regions and returns rects in image pixel coordinates. + /// - Parameter image: The image to analyze. + /// - Returns: Pixel-space rects (top-left origin), possibly empty. + func detectPrivacyRegions(in image: CGImage) async throws -> [CGRect] { + let engine = ScreenCoderEngine.shared + guard let provider = try? await engine.currentProvider() else { + throw PrivacyDetectionError.providerNotConfigured + } + guard await provider.isAvailable else { + throw PrivacyDetectionError.providerNotConfigured + } + + logger.info("Smart privacy detection started via \(provider.name)") + let result = try await provider.analyze(image: image, prompt: .privacy) + + let width = CGFloat(image.width) + let height = CGFloat(image.height) + let rects = result.segments + .filter { $0.confidence >= 0.5 } + .compactMap { segment -> CGRect? in + let rect = CGRect( + x: segment.boundingBox.origin.x * width, + y: segment.boundingBox.origin.y * height, + width: segment.boundingBox.width * width, + height: segment.boundingBox.height * height + ) + // Discard degenerate boxes + guard rect.width >= 4, rect.height >= 4 else { return nil } + return rect.integral + } + logger.info("Smart privacy detection found \(rects.count) regions") + return rects + } +} diff --git a/TransFrame/Services/VLMProvider.swift b/TransFrame/Services/VLMProvider.swift index 94afe11..32ac43f 100644 --- a/TransFrame/Services/VLMProvider.swift +++ b/TransFrame/Services/VLMProvider.swift @@ -54,6 +54,56 @@ protocol VLMProvider: Sendable { /// - Returns: Analysis result containing text segments with positions /// - Throws: VLMProviderError if analysis fails func analyze(image: CGImage) async throws -> ScreenAnalysisResult + + /// Analyze an image with a custom prompt while reusing the standard + /// segments JSON response format. Providers that support prompt + /// customization override this; the default falls back to the OCR prompt. + /// - Parameters: + /// - image: The image to analyze + /// - prompt: The system/user prompt pair to send + /// - Returns: Analysis result containing segments with positions + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult +} + +extension VLMProvider { + func analyze(image: CGImage, prompt: VLMPromptPair) async throws -> ScreenAnalysisResult { + try await analyze(image: image) + } +} + +// MARK: - Custom Prompt Pair + +/// A system/user prompt pair for custom VLM analysis tasks. +struct VLMPromptPair: Sendable, Equatable { + let system: String + let user: String + + /// The standard OCR text-extraction prompt + static let ocr = VLMPromptPair( + system: VLMPromptTemplate.systemPrompt, + user: VLMPromptTemplate.userPrompt + ) + + /// Privacy-region detection prompt: returns sensitive regions + /// (phone numbers, passwords, amounts, faces, keys, emails, IDs) + /// in the same segments JSON format. + static let privacy = VLMPromptPair( + system: """ + You are a privacy detection assistant. Locate regions containing sensitive personal information in the image: phone numbers, passwords, amounts/prices, faces, API keys/tokens, emails, and ID/card numbers. + Return JSON: {"segments":[{"text":"","boundingBox":{"x":0.0,"y":0.0,"width":0.0,"height":0.0},"confidence":0.9}]} + """, + user: """ + Detect all sensitive privacy regions in this image (phone numbers, passwords, amounts, faces, API keys/tokens, emails, ID numbers). + Return each region in JSON format: + {"segments":[{"text":"","boundingBox":{"x":0.0,"y":0.0,"width":0.0,"height":0.0},"confidence":0.9}]} + + - text: the category of the sensitive content (e.g. "phone", "face", "email") + - x,y: top-left corner (0.0-1.0 normalized to image size) + - width,height: box dimensions (0.0-1.0), make boxes tightly cover the content + - confidence: 0.0-1.0 + If nothing sensitive is found, return {"segments":[]}. + """ + ) } // MARK: - VLM Provider Errors diff --git a/TransFrameTests/AnalysisImageEncoderTests.swift b/TransFrameTests/AnalysisImageEncoderTests.swift new file mode 100644 index 0000000..4d5d154 --- /dev/null +++ b/TransFrameTests/AnalysisImageEncoderTests.swift @@ -0,0 +1,68 @@ +import CoreGraphics +import XCTest +@testable import TransFrame + +/// Tests for the shared analysis image encoder: Retina captures must be +/// downsampled before upload, small images must pass through untouched. +final class AnalysisImageEncoderTests: XCTestCase { + + func testDownsampleShrinksLongestEdgeAndKeepsAspectRatio() { + let image = Self.makeImage(width: 4000, height: 3000) + let downsampled = AnalysisImageEncoder.downsampled(image) + + XCTAssertEqual(downsampled.width, 2000, "longest edge must cap at maxDimension") + XCTAssertEqual(downsampled.height, 1500, "aspect ratio must be preserved") + } + + func testDownsampleLeavesSmallImagesUnchanged() { + let image = Self.makeImage(width: 800, height: 600) + let downsampled = AnalysisImageEncoder.downsampled(image, maxDimension: 2000) + + XCTAssertTrue(downsampled === image, "images within the limit must not be re-encoded") + } + + func testDownsampleHandlesExtremeAspectRatios() { + // 3000px wide strip capped to 2000 -> height stays >= 1 + let strip = Self.makeImage(width: 3000, height: 8) + let downsampled = AnalysisImageEncoder.downsampled(strip) + + XCTAssertEqual(downsampled.width, 2000) + XCTAssertGreaterThanOrEqual(downsampled.height, 1) + } + + func testJPEGEncodingProducesDataURI() throws { + let image = Self.makeImage(width: 640, height: 480) + let encoded = try AnalysisImageEncoder.jpeg(image) + + XCTAssertFalse(encoded.data.isEmpty) + XCTAssertEqual(encoded.mimeType, "image/jpeg") + XCTAssertTrue(encoded.dataURI.hasPrefix("data:image/jpeg;base64,")) + XCTAssertTrue(encoded.dataURI.hasSuffix(encoded.base64String)) + } + + func testPNGEncodingProducesPNGSignature() throws { + let image = Self.makeImage(width: 640, height: 480) + let encoded = try AnalysisImageEncoder.png(image) + + XCTAssertGreaterThan(encoded.data.count, 8) + XCTAssertEqual(Array(encoded.data.prefix(4)), [0x89, 0x50, 0x4E, 0x47], "PNG magic bytes expected") + XCTAssertEqual(encoded.mimeType, "image/png") + } + + // MARK: - Utilities + + private static func makeImage(width: Int, height: Int) -> CGImage { + let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + )! + context.setFillColor(CGColor(red: 0.2, green: 0.4, blue: 0.6, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + return context.makeImage()! + } +} diff --git a/TransFrameTests/CaptureWorkflowTests.swift b/TransFrameTests/CaptureWorkflowTests.swift index ff72b54..db171c4 100644 --- a/TransFrameTests/CaptureWorkflowTests.swift +++ b/TransFrameTests/CaptureWorkflowTests.swift @@ -180,4 +180,290 @@ final class CaptureWorkflowTests: XCTestCase { // Outside XCTAssertFalse(redactionAnn.hitTest(point: CGPoint(x: 100, y: 100), tolerance: 2)) } + + func testRetinaAnnotationScalingAndNumberLabelCircleRect() { + let label = NumberLabelAnnotation( + position: CGPoint(x: 50, y: 60), + number: 1, + size: 26, + color: .red, + leadPoint: CGPoint(x: 120, y: 140) + ) + // circleRect must stay a 26x26 square centered at position even when leadPoint expands bounds + XCTAssertEqual(label.circleRect, CGRect(x: 37, y: 47, width: 26, height: 26)) + XCTAssertGreaterThan(label.bounds.width, 26) + + let scaled = Annotation.numberLabel(label).offsetBy(dx: -10, dy: -20).scaled(by: 2.0) + guard case .numberLabel(let scaledLabel) = scaled else { + XCTFail("Expected scaled numberLabel") + return + } + XCTAssertEqual(scaledLabel.position, CGPoint(x: 80, y: 80)) + XCTAssertEqual(scaledLabel.size, 52) + XCTAssertEqual(scaledLabel.leadPoint, CGPoint(x: 220, y: 240)) + } + + func testAnnotationHandlesAndResize() { + let rectAnno = Annotation.rectangle( + RectangleAnnotation(rect: CGRect(x: 10, y: 20, width: 100, height: 80)) + ) + // Hit bottom-right handle (110 + 4, 100 + 4) + let hitHandle = rectAnno.hitTestHandle(at: CGPoint(x: 114, y: 104), padding: 4, tolerance: 10) + XCTAssertEqual(hitHandle, .bottomRight) + + let resized = rectAnno.applyingHandleDrag(.bottomRight, to: CGPoint(x: 160, y: 140)) + if case .rectangle(let r) = resized { + XCTAssertEqual(r.rect, CGRect(x: 10, y: 20, width: 150, height: 120)) + } else { + XCTFail("Expected rectangle annotation") + } + + // Arrow curve control handle + let arrowAnno = Annotation.arrow( + ArrowAnnotation(startPoint: CGPoint(x: 0, y: 0), endPoint: CGPoint(x: 100, y: 0)) + ) + let curvedArrow = arrowAnno.applyingHandleDrag(.controlPoint, to: CGPoint(x: 50, y: 40)) + if case .arrow(let a) = curvedArrow { + XCTAssertEqual(a.controlPoint, CGPoint(x: 50, y: 40)) + } else { + XCTFail("Expected arrow annotation") + } + } + + @MainActor + func testPreviewViewModelDragUndoAndNumberLabelLeadPointStability() { + let baseImage = createTestImage(width: 200, height: 200, color: .white) + let display = DisplayInfo(id: 1, name: "Test", frame: CGRect(x: 0, y: 0, width: 200, height: 200), scaleFactor: 2.0, isPrimary: true) + let label = NumberLabelAnnotation( + position: CGPoint(x: 40, y: 40), + number: 1, + size: 24, + color: .red, + leadPoint: CGPoint(x: 80, y: 80) + ) + let screenshot = Screenshot(image: baseImage, sourceDisplay: display, annotations: [.numberLabel(label)]) + let vm = PreviewViewModel(screenshot: screenshot) + + vm.selectAnnotation(at: 0) + vm.beginDraggingAnnotation(at: CGPoint(x: 40, y: 40)) + // Multiple drag steps should offset relative to initial position, not accumulate exponentially + vm.continueDraggingAnnotation(to: CGPoint(x: 50, y: 50)) + vm.continueDraggingAnnotation(to: CGPoint(x: 60, y: 70)) + vm.endDraggingAnnotation() + + guard case .numberLabel(let movedLabel) = vm.annotations[0] else { + XCTFail("Expected numberLabel") + return + } + XCTAssertEqual(movedLabel.position, CGPoint(x: 60, y: 70)) + XCTAssertEqual(movedLabel.leadPoint, CGPoint(x: 100, y: 110)) + XCTAssertTrue(vm.canUndo) + + vm.undo() + guard case .numberLabel(let restoredLabel) = vm.annotations[0] else { + XCTFail("Expected numberLabel after undo") + return + } + XCTAssertEqual(restoredLabel.position, CGPoint(x: 40, y: 40)) + XCTAssertEqual(restoredLabel.leadPoint, CGPoint(x: 80, y: 80)) + } + + @MainActor + func testPreviewViewModelNonDestructiveTextEditing() { + let baseImage = createTestImage(width: 200, height: 200, color: .white) + let display = DisplayInfo(id: 1, name: "Test", frame: CGRect(x: 0, y: 0, width: 200, height: 200), scaleFactor: 1.0, isPrimary: true) + let textAnno = Annotation.text(TextAnnotation(position: CGPoint(x: 20, y: 20), content: "Original")) + let rectAnno = Annotation.rectangle(RectangleAnnotation(rect: CGRect(x: 50, y: 50, width: 40, height: 40))) + let screenshot = Screenshot(image: baseImage, sourceDisplay: display, annotations: [textAnno, rectAnno]) + let vm = PreviewViewModel(screenshot: screenshot) + + // Start editing first annotation and cancel -> must preserve original text and Z-Order (index 0) + vm.beginEditingTextAnnotation(at: 0) + XCTAssertTrue(vm.isWaitingForTextInput) + vm.textInputContent = "Draft Change" + vm.cancelCurrentDrawing() + + XCTAssertEqual(vm.annotations.count, 2) + if case .text(let preserved) = vm.annotations[0] { + XCTAssertEqual(preserved.content, "Original") + } else { + XCTFail("Text annotation lost or reordered on cancel") + } + + // Now edit and commit -> updates in-place at index 0 and pushes undo state + vm.beginEditingTextAnnotation(at: 0) + vm.textInputContent = "Updated Text" + vm.commitTextInput() + + if case .text(let updated) = vm.annotations[0] { + XCTAssertEqual(updated.content, "Updated Text") + } else { + XCTFail("Expected updated text at index 0") + } + XCTAssertTrue(vm.canUndo) + } + + func testPrivacyRedactionShortcutDoesNotConflictWithPin() { + XCTAssertEqual(AnnotationToolType.privacyRedaction.keyboardShortcut, "x") + XCTAssertNotEqual(AnnotationToolType.privacyRedaction.keyboardShortcut, "p") + } + + func testShapeFillColorSeparationAndLegacyFallback() { + // Legacy annotation without fillColor falls back to stroke color + let legacy = RectangleAnnotation(rect: CGRect(x: 0, y: 0, width: 10, height: 10)) + XCTAssertNil(legacy.fillColor) + XCTAssertEqual(legacy.effectiveFillColor, legacy.style.color) + + // Independent fill color + let customFill = CodableColor(red: 0.2, green: 0.6, blue: 1.0) + let filled = RectangleAnnotation( + rect: CGRect(x: 0, y: 0, width: 10, height: 10), + style: StrokeStyle(color: .red, lineWidth: 2), + isFilled: true, + fillColor: customFill + ) + XCTAssertEqual(filled.effectiveFillColor, customFill) + XCTAssertEqual(filled.style.color, .red) + + // Ellipse mirrors the same behavior + let ellipse = Annotation.ellipse(EllipseAnnotation(rect: CGRect(x: 0, y: 0, width: 10, height: 10))) + let updated = ellipse.updatingFillColor(customFill) + guard case .ellipse(let e) = updated else { + XCTFail("Expected ellipse") + return + } + XCTAssertEqual(e.fillColor, customFill) + + // Non-shape types ignore fill updates + let lineAnno = Annotation.line(LineAnnotation(startPoint: .zero, endPoint: CGPoint(x: 10, y: 10))) + if case .line(let l) = lineAnno.updatingFillColor(customFill) { + XCTAssertEqual(l.style.color, StrokeStyle.default.color) + } else { + XCTFail("Expected line") + } + } + + func testTextWrapWidthMultiLineBoundsAndScaling() { + let content = "Hello world this is a multi-line annotation test" + let single = TextAnnotation(position: .zero, content: content) + let wrapped = TextAnnotation(position: .zero, content: content, wrapWidth: 60) + + // Wrapping must shrink width but grow height relative to the single-line case + XCTAssertLessThan(wrapped.bounds.width, single.bounds.width) + XCTAssertGreaterThan(wrapped.bounds.height, single.bounds.height) + XCTAssertGreaterThanOrEqual(wrapped.bounds.height, single.style.fontSize * 1.5) + + // Hard line breaks count as extra lines + let multiline = TextAnnotation(position: .zero, content: "one\ntwo\nthree") + XCTAssertGreaterThan(multiline.bounds.height, multiline.style.fontSize * 2) + + // Scaling preserves wrapWidth proportionally + let scaled = Annotation.text(multiline).scaled(by: 2.0) + guard case .text(let t) = scaled else { + XCTFail("Expected text") + return + } + XCTAssertEqual(t.style.fontSize, multiline.style.fontSize * 2.0, accuracy: 0.01) + XCTAssertNil(multiline.wrapWidth) + + let wrappedScaled = Annotation.text(wrapped).scaled(by: 2.0) + guard case .text(let tw) = wrappedScaled else { + XCTFail("Expected text") + return + } + XCTAssertEqual(tw.wrapWidth ?? 0, 120, accuracy: 0.5) + + // Offset moves the whole box + let moved = Annotation.text(wrapped).offsetBy(dx: 30, dy: 15) + guard case .text(let tm) = moved else { + XCTFail("Expected text") + return + } + XCTAssertEqual(tm.position, CGPoint(x: 30, y: 15)) + } + + func testHandleDragPreservesAspectRatioWhenRequested() { + let rectAnno = Annotation.rectangle( + RectangleAnnotation(rect: CGRect(x: 0, y: 0, width: 100, height: 50)) + ) + + // Free drag changes aspect + let free = rectAnno.applyingHandleDrag(.bottomRight, to: CGPoint(x: 200, y: 80)) + guard case .rectangle(let r) = free else { + XCTFail("Expected rectangle") + return + } + XCTAssertEqual(r.rect.width, 200, accuracy: 0.5) + XCTAssertEqual(r.rect.height, 80, accuracy: 0.5) + + // ⌘ drag locks the 2:1 ratio of the original + let locked = rectAnno.applyingHandleDrag( + .bottomRight, + to: CGPoint(x: 200, y: 80), + options: HandleDragOptions(preserveAspectRatio: true) + ) + guard case .rectangle(let lr) = locked else { + XCTFail("Expected rectangle") + return + } + XCTAssertEqual(lr.rect.width / lr.rect.height, 2.0, accuracy: 0.02) + XCTAssertEqual(lr.rect.origin, .zero) + + // Top-left handle keeps the opposite corner anchored + let fromTopLeft = rectAnno.applyingHandleDrag(.topLeft, to: CGPoint(x: -10, y: -10)) + guard case .rectangle(let tl) = fromTopLeft else { + XCTFail("Expected rectangle") + return + } + XCTAssertEqual(tl.rect.maxX, 100, accuracy: 0.5) + XCTAssertEqual(tl.rect.maxY, 50, accuracy: 0.5) + } + + func testTextHandlesCornerScaleAndEdgeWrap() { + let anno = TextAnnotation( + position: CGPoint(x: 0, y: 0), + content: "Hello annotation", + style: TextStyle(color: .red, fontSize: 20, fontName: ".AppleSystemUIFont"), + wrapWidth: 200 + ) + let base = Annotation.text(anno) + + // Eight handles around the text box (4 corners + 4 edges) + let handles = base.controlHandles() + XCTAssertEqual(handles.count, 8) + XCTAssertTrue(handles.map(\.handle).contains(.leftEdge)) + XCTAssertTrue(handles.map(\.handle).contains(.rightEdge)) + XCTAssertTrue(handles.map(\.handle).contains(.topEdge)) + XCTAssertTrue(handles.map(\.handle).contains(.bottomEdge)) + + // Corner drag scales font size and wrap width together + let originalHeight = anno.bounds.height + let scaled = base.applyingHandleDrag(.bottomRight, to: CGPoint(x: anno.bounds.maxX * 2, y: anno.bounds.maxY * 2)) + guard case .text(let st) = scaled else { + XCTFail("Expected text") + return + } + XCTAssertGreaterThan(st.style.fontSize, 20) + XCTAssertGreaterThan(st.wrapWidth ?? 0, 200) + XCTAssertGreaterThan(st.bounds.height, originalHeight) + + // Right edge drag changes wrap width but keeps the font size + let wrapped = base.applyingHandleDrag(.rightEdge, to: CGPoint(x: 60, y: anno.bounds.midY)) + guard case .text(let wt) = wrapped else { + XCTFail("Expected text") + return + } + XCTAssertEqual(wt.style.fontSize, 20, accuracy: 0.01) + XCTAssertEqual(wt.wrapWidth ?? 0, 60, accuracy: 1.0) + XCTAssertLessThan(wt.bounds.width, anno.bounds.width) + + // Left edge drag keeps the right edge fixed + let fromLeft = base.applyingHandleDrag(.leftEdge, to: CGPoint(x: 40, y: anno.bounds.midY)) + guard case .text(let lt) = fromLeft else { + XCTFail("Expected text") + return + } + let rightEdgeBefore = anno.bounds.maxX + XCTAssertEqual(lt.bounds.maxX, rightEdgeBefore, accuracy: 2.0) + } } diff --git a/TransFrameTests/GLMOCRVLMProviderTests.swift b/TransFrameTests/GLMOCRVLMProviderTests.swift deleted file mode 100644 index 438a375..0000000 --- a/TransFrameTests/GLMOCRVLMProviderTests.swift +++ /dev/null @@ -1,226 +0,0 @@ -import CoreGraphics -import XCTest -@testable import TransFrame - -final class GLMOCRVLMProviderTests: XCTestCase { - func testParseResponseMapsLayoutDetailsToScreenSegments() throws { - let json = """ - { - "id": "task_123", - "model": "GLM-OCR", - "layout_details": [ - [ - { - "index": 1, - "label": "text", - "bbox_2d": [0.1, 0.2, 0.4, 0.3], - "content": "Hello world", - "width": 1200, - "height": 800 - }, - { - "index": 2, - "label": "table", - "bbox_2d": [0.5, 0.2, 0.9, 0.6], - "content": "
Total42
", - "width": 1200, - "height": 800 - }, - { - "index": 3, - "label": "image", - "bbox_2d": [0.0, 0.0, 0.2, 0.2], - "content": "https://example.com/image.png", - "width": 1200, - "height": 800 - } - ] - ], - "data_info": { - "num_pages": 1, - "pages": [ - { - "width": 1200, - "height": 800 - } - ] - } - } - """ - - let result = try GLMOCRVLMProvider.parseResponse( - Data(json.utf8), - fallbackImageSize: CGSize(width: 100, height: 100) - ) - - XCTAssertEqual(result.imageSize, CGSize(width: 1200, height: 800)) - XCTAssertEqual(result.segments.map(\.text), ["Hello world", "Total 42"]) - XCTAssertEqual(result.segments.count, 2) - guard result.segments.count == 2 else { return } - assertRect(result.segments[0].boundingBox, equals: CGRect(x: 0.1, y: 0.2, width: 0.3, height: 0.1)) - assertRect(result.segments[1].boundingBox, equals: CGRect(x: 0.5, y: 0.2, width: 0.4, height: 0.4)) - } - - func testParseLocalResponseMapsOpenAICompatibleContentToScreenSegments() throws { - let json = """ - { - "choices": [ - { - "message": { - "content": "{\\"segments\\":[{\\"text\\":\\"Local OCR\\",\\"boundingBox\\":{\\"x\\":0.2,\\"y\\":0.3,\\"width\\":0.4,\\"height\\":0.1},\\"confidence\\":0.98}]}" - } - } - ] - } - """ - - let result = try GLMOCRVLMProvider.parseLocalResponse( - Data(json.utf8), - fallbackImageSize: CGSize(width: 640, height: 480) - ) - - XCTAssertEqual(result.imageSize, CGSize(width: 640, height: 480)) - XCTAssertEqual(result.segments.map(\.text), ["Local OCR"]) - XCTAssertEqual(result.segments.count, 1) - guard result.segments.count == 1 else { return } - assertRect(result.segments[0].boundingBox, equals: CGRect(x: 0.2, y: 0.3, width: 0.4, height: 0.1)) - } - - func testParseLocalResponseSupportsTextOnlyPayload() throws { - let json = #""" - { - "choices": [ - { - "message": { - "content": "{\"Text\":\"返回 200\\n-scheme TransFrame -destination 'platform=macOS'\"}" - } - } - ] - } - """# - - let result = try GLMOCRVLMProvider.parseLocalResponse( - Data(json.utf8), - fallbackImageSize: CGSize(width: 640, height: 480) - ) - - XCTAssertEqual( - result.segments.map(\.text), - ["返回 200", "-scheme TransFrame -destination 'platform=macOS'"] - ) - } - - private func assertRect(_ actual: CGRect, equals expected: CGRect, accuracy: CGFloat = 0.0001) { - XCTAssertEqual(actual.origin.x, expected.origin.x, accuracy: accuracy) - XCTAssertEqual(actual.origin.y, expected.origin.y, accuracy: accuracy) - XCTAssertEqual(actual.size.width, expected.size.width, accuracy: accuracy) - XCTAssertEqual(actual.size.height, expected.size.height, accuracy: accuracy) - } -} - -final class OpenAIVLMProviderTests: XCTestCase { - var provider: OpenAIVLMProvider! - - override func setUp() { - super.setUp() - let config = VLMProviderConfiguration(apiKey: "test-key", baseURL: URL(string: "https://api.openai.com/v1")!, modelName: "gpt-4o") - provider = OpenAIVLMProvider(configuration: config) - } - - func testExtractContentAndStatusWithReasoningContentFallback() throws { - let json = """ - { - "choices": [ - { - "message": { - "role": "assistant", - "content": "", - "reasoning_content": "{\\"segments\\":[]}" - }, - "finish_reason": "stop" - } - ] - } - """ - let data = Data(json.utf8) - let (content, isTruncated, finishReason) = try provider.extractContentAndStatus(from: data) - XCTAssertEqual(content, "{\"segments\":[]}") - XCTAssertFalse(isTruncated) - XCTAssertEqual(finishReason, "stop") - } - - func testExtractContentAndStatusWithReasoningFallback() throws { - let json = """ - { - "choices": [ - { - "message": { - "role": "assistant", - "content": " ", - "reasoning": "{\\"segments\\":[]}" - }, - "finish_reason": "stop" - } - ] - } - """ - let data = Data(json.utf8) - let (content, isTruncated, finishReason) = try provider.extractContentAndStatus(from: data) - XCTAssertEqual(content, "{\"segments\":[]}") - XCTAssertFalse(isTruncated) - XCTAssertEqual(finishReason, "stop") - } - - func testExtractContentManuallyWithReasoningFallback() throws { - // Broken JSON structure but has reasoning_content - let brokenJSON = """ - { - "choices": [ - { - "message": { - "role": "assistant", - "content": null, - "reasoning_content": "{\\"segments\\":[]}" - } - """ - let extracted = provider.extractContentManually(from: brokenJSON) - XCTAssertEqual(extracted, "{\"segments\":[]}") - - // Broken JSON structure but has reasoning - let brokenJSON2 = """ - { - "choices": [ - { - "message": { - "role": "assistant", - "content": "", - "reasoning": "{\\"segments\\":[]}" - } - """ - let extracted2 = provider.extractContentManually(from: brokenJSON2) - XCTAssertEqual(extracted2, "{\"segments\":[]}") - } - - func testParseVLMContentWithEmptyResponseError() { - XCTAssertThrowsError(try provider.parseVLMContent(" ")) { error in - guard let providerError = error as? VLMProviderError else { - XCTFail("Expected VLMProviderError") - return - } - XCTAssertTrue(providerError.localizedDescription.contains("Received empty response from model")) - } - } - - func testParseVLMContentWithCleanedEmptyError() { - // A response that has code blocks but nothing inside - let content = "```json\n```" - XCTAssertThrowsError(try provider.parseVLMContent(content)) { error in - guard let providerError = error as? VLMProviderError else { - XCTFail("Expected VLMProviderError") - return - } - XCTAssertTrue(providerError.localizedDescription.contains("Cleaned content is empty")) - XCTAssertTrue(providerError.localizedDescription.contains("```json")) - } - } -} diff --git a/TransFrameTests/PRDAcceptanceTests.swift b/TransFrameTests/PRDAcceptanceTests.swift new file mode 100644 index 0000000..e77d49f --- /dev/null +++ b/TransFrameTests/PRDAcceptanceTests.swift @@ -0,0 +1,785 @@ +import XCTest +import CoreGraphics +import AppKit +@testable import TransFrame + +/// PRD acceptance tests for 标注体验与内嵌面板增强. +/// Each test maps to a PRD success criterion / business rule. +@MainActor +final class PRDAcceptanceTests: XCTestCase { + + // MARK: - Helpers + + private func stringsURL(_ lang: String) -> URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // TransFrameTests/ + .deletingLastPathComponent() // repo root + .appendingPathComponent("TransFrame/Resources/\(lang).lproj/Localizable.strings") + } + + private func parseStrings(_ lang: String) -> [String: String] { + let contents = try! String(contentsOf: stringsURL(lang), encoding: .utf8) + var result: [String: String] = [:] + // Matches: "key" = "value"; (value may contain escaped quotes) + let pattern = "\"((?:[^\"\\\\]|\\\\.)+)\"\\s*=\\s*\"((?:[^\"\\\\]|\\\\.)+)\"\\s*;" + let regex = try! NSRegularExpression(pattern: pattern) + let ns = contents as NSString + for match in regex.matches(in: contents, range: NSRange(location: 0, length: ns.length)) { + let key = ns.substring(with: match.range(at: 1)) + let value = ns.substring(with: match.range(at: 2)) + result[key] = value + } + return result + } + + private func sourceLocalizationKeys() -> Set { + var keys = Set() + let root = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("TransFrame") + let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: nil)! + let patterns = [ + "String\\(localized:\\s*\"([^\"]+)\"", + "NSLocalizedString\\(\"([^\"]+)\"", + "Text\\(\"([a-zA-Z0-9_.]+)\"\\)" // dot-separated SwiftUI keys only + ] + for case let file as URL in enumerator where file.pathExtension == "swift" { + guard let contents = try? String(contentsOf: file, encoding: .utf8) else { continue } + for pattern in patterns { + let regex = try! NSRegularExpression(pattern: pattern) + let ns = contents as NSString + for match in regex.matches(in: contents, range: NSRange(location: 0, length: ns.length)) { + let key = ns.substring(with: match.range(at: 1)) + // Only treat dotted identifiers as localization keys + if key.contains(".") { + keys.insert(key) + } + } + } + } + return keys + } + + // MARK: - 5.1 Menu bar localization (success criterion 1) + + /// Every localization key referenced in source must exist in BOTH language + /// files with a non-empty, non-raw-key value. Catches "raw key exposed in UI". + func testAllLocalizationKeysResolvedInBothLanguages() { + let zh = parseStrings("zh-Hans") + let en = parseStrings("en") + let keys = sourceLocalizationKeys() + XCTAssertGreaterThan(keys.count, 50, "Expected to scan a meaningful number of keys") + + var missing: [String] = [] + var rawExposed: [String] = [] + for key in keys.sorted() { + for (lang, table) in [("zh-Hans", zh), ("en", en)] { + if table[key] == nil { + missing.append("\(lang): \(key)") + } else if table[key] == key { + rawExposed.append("\(lang): \(key)") + } + } + } + XCTAssertTrue(missing.isEmpty, "Missing localization keys:\n" + missing.joined(separator: "\n")) + XCTAssertTrue(rawExposed.isEmpty, "Values equal to key (raw key exposure):\n" + rawExposed.joined(separator: "\n")) + } + + /// PRD 5.1: the two previously broken menu entries must resolve in both languages. + func testMenuColorPickerAndOpenImageLocalized() { + let zh = parseStrings("zh-Hans") + let en = parseStrings("en") + + XCTAssertEqual(zh["menu.pick.color"], "屏幕取色") + XCTAssertEqual(en["menu.pick.color"], "Pick Screen Color") + XCTAssertEqual(zh["menu.open.image"], "打开图片编辑...") + XCTAssertEqual(en["menu.open.image"], "Open Image for Editing...") + + for key in ["menu.capture.full.screen", "menu.capture.selection", "menu.translation.mode", + "menu.translation.history", "menu.settings", "menu.about", "menu.quit"] { + XCTAssertNotNil(zh[key], "zh missing \(key)") + XCTAssertNotNil(en[key], "en missing \(key)") + } + } + + // MARK: - 5.1 Text tool cognition (success criterion 3) + + func testTextToolDisplayNameAndTooltip() { + XCTAssertEqual(AnnotationToolType.text.displayName, "文本") + let zh = parseStrings("zh-Hans") + XCTAssertEqual(zh["tool.text"], "文本") + XCTAssertEqual(parseStrings("en")["tool.text"], "Text") + + // In-place toolbar shares the same display name (tooltip = name (shortcut)) + let inPlace = SelectionInPlaceToolbar.AnnotationTool.text + XCTAssertEqual(inPlace.displayName, "文本") + XCTAssertEqual(inPlace.shortcutKey, "T") + XCTAssertTrue(inPlace.tooltip.contains("文本")) + XCTAssertTrue(inPlace.tooltip.contains("T")) + } + + // MARK: - 5.3 Line tool (success criterion 4) + + func testLineToolExistsInBothToolchains() { + // Preview tool chain + XCTAssertTrue(AnnotationToolType.allCases.contains(.line)) + XCTAssertEqual(AnnotationToolType.line.displayName, "直线") + + // In-place toolbar must expose the line tool as well + XCTAssertTrue(SelectionInPlaceToolbar.AnnotationTool.allCases.contains(.line)) + XCTAssertEqual(SelectionInPlaceToolbar.AnnotationTool.line.displayName, "直线") + + // Drawing produces a line annotation between the two drag points + var tool = LineTool() + tool.beginDrawing(at: CGPoint(x: 10, y: 10)) + tool.continueDrawing(to: CGPoint(x: 100, y: 60)) + let anno = tool.endDrawing(at: CGPoint(x: 100, y: 60)) + guard case .line(let line)? = anno else { + XCTFail("Expected line annotation") + return + } + XCTAssertEqual(line.startPoint, CGPoint(x: 10, y: 10)) + XCTAssertEqual(line.endPoint, CGPoint(x: 100, y: 60)) + } + + // MARK: - 5.3 ⌘ square/circle + aspect lock (success criterion 5) + + func testCommandConstrainsSquareAndCircleWhileDrawing() { + // Pure geometry helper + let square = GeometryConstraint.squareRect(from: CGPoint(x: 10, y: 10), to: CGPoint(x: 80, y: 30)) + XCTAssertEqual(square.width, square.height) + XCTAssertEqual(square.width, 70, accuracy: 0.01) + + // Rectangle tool + var rect = RectangleTool() + rect.beginDrawing(at: CGPoint(x: 10, y: 10)) + rect.continueDrawing(to: CGPoint(x: 80, y: 30)) + let freeRect = rect.calculatedRect(constrainSquare: false) + XCTAssertEqual(freeRect, CGRect(x: 10, y: 10, width: 70, height: 20)) + let lockedRect = rect.calculatedRect(constrainSquare: true) + XCTAssertEqual(lockedRect.width, lockedRect.height) + + // Ellipse tool (circle) + var ellipse = EllipseTool() + ellipse.beginDrawing(at: CGPoint(x: 0, y: 0)) + ellipse.continueDrawing(to: CGPoint(x: 50, y: 20)) + let lockedEllipse = ellipse.calculatedRect(constrainSquare: true) + XCTAssertEqual(lockedEllipse.width, lockedEllipse.height) + let freeEllipse = ellipse.calculatedRect(constrainSquare: false) + XCTAssertEqual(freeEllipse.size, CGSize(width: 50, height: 20)) + } + + func testCommandLocksAspectRatioWhileResizing() { + let anno = Annotation.ellipse( + EllipseAnnotation(rect: CGRect(x: 0, y: 0, width: 200, height: 100)) + ) + let locked = anno.applyingHandleDrag( + .bottomRight, + to: CGPoint(x: 400, y: 350), + options: HandleDragOptions(preserveAspectRatio: true) + ) + guard case .ellipse(let e) = locked else { + XCTFail("Expected ellipse") + return + } + XCTAssertEqual(e.rect.width / e.rect.height, 2.0, accuracy: 0.02) + } + + // MARK: - 5.3 Dual-color shapes (success criterion 6) + + func testBorderAndFillColorsAreIndependent() { + let border = CodableColor(.red) + let fill = CodableColor(red: 0, green: 0.5, blue: 1, opacity: 0.6) + + let rect = RectangleAnnotation( + rect: CGRect(x: 0, y: 0, width: 10, height: 10), + style: StrokeStyle(color: border, lineWidth: 3), + isFilled: true, + fillColor: fill + ) + XCTAssertEqual(rect.style.color, border) + XCTAssertEqual(rect.effectiveFillColor, fill) + XCTAssertNotEqual(rect.style.color, rect.effectiveFillColor) + + // Hollow: background transparent (fill not applied by renderers when !isFilled) + let hollow = RectangleAnnotation(rect: CGRect(x: 0, y: 0, width: 10, height: 10), isFilled: false) + XCTAssertFalse(hollow.isFilled) + + // Ellipse mirrors the same independence + let ellipse = EllipseAnnotation( + rect: CGRect(x: 0, y: 0, width: 10, height: 10), + style: StrokeStyle(color: border, lineWidth: 2), + isFilled: true, + fillColor: fill + ) + XCTAssertEqual(ellipse.style.color, border) + XCTAssertEqual(ellipse.effectiveFillColor, fill) + + // updatingFillColor only touches the background, never the border + let updated = Annotation.rectangle(rect).updatingFillColor(CodableColor(.green)) + guard case .rectangle(let r) = updated else { + XCTFail("Expected rectangle") + return + } + XCTAssertEqual(r.effectiveFillColor, CodableColor(.green)) + XCTAssertEqual(r.style.color, border, "Border color must stay untouched") + } + + // MARK: - 5.2 Text box multi-line / drag / handles (success criterion 2) + + func testTextBoxCommitSelectsAndExposesEightHandles() { + let image = Self.makeImage(width: 100, height: 100) + let display = DisplayInfo(id: 1, name: "T", frame: CGRect(x: 0, y: 0, width: 100, height: 100), scaleFactor: 1.0, isPrimary: true) + let vm = PreviewViewModel(screenshot: Screenshot(image: image, sourceDisplay: display)) + + vm.selectTool(.text) + vm.beginDrawing(at: CGPoint(x: 5, y: 5)) + vm.textInputContent = "Hello\nWorld multiline" + vm.commitTextInput() + + // Committed text is selected so its handles show immediately + XCTAssertEqual(vm.selectedAnnotationIndex, 0) + XCTAssertEqual(vm.annotations.count, 1) + + let handles = vm.annotations[0].controlHandles() + XCTAssertEqual(handles.count, 8, "Text box must expose 8 control handles (4 corners + 4 edges)") + XCTAssertTrue(handles.map(\.handle).contains(.leftEdge)) + XCTAssertTrue(handles.map(\.handle).contains(.rightEdge)) + + // Whole-box drag moves content without reflow + let before = vm.annotations[0].bounds + vm.beginDraggingAnnotation(at: before.origin) + vm.continueDraggingAnnotation(to: CGPoint(x: before.minX + 25, y: before.minY + 15)) + vm.endDraggingAnnotation() + let after = vm.annotations[0].bounds + XCTAssertEqual(after.origin, CGPoint(x: before.minX + 25, y: before.minY + 15)) + XCTAssertEqual(after.size, before.size, "Dragging must not reflow or rescale content") + } + + // MARK: - 5.4 Mosaic intensity (success criterion 8) + + func testMosaicIntensityClampingAndLiveResize() { + // Block size is clamped to a sane range (granularity control) + XCTAssertEqual(MosaicAnnotation(rect: CGRect(x: 0, y: 0, width: 20, height: 20), blockSize: 2).blockSize, 4) + XCTAssertEqual(MosaicAnnotation(rect: CGRect(x: 0, y: 0, width: 20, height: 20), blockSize: 999).blockSize, 128) + + // Intensity presets offered in the in-place style bar map to distinct sizes + let light = 4, medium = 10, strong = 20 + XCTAssertLessThan(light, medium) + XCTAssertLessThan(medium, strong) + + // Selected mosaic can be re-strengthened + let image = Self.makeImage(width: 60, height: 60) + let display = DisplayInfo(id: 1, name: "T", frame: CGRect(x: 0, y: 0, width: 60, height: 60), scaleFactor: 1.0, isPrimary: true) + var screenshot = Screenshot(image: image, sourceDisplay: display) + screenshot = screenshot.adding(.mosaic(MosaicAnnotation(rect: CGRect(x: 5, y: 5, width: 30, height: 30), blockSize: 4))) + let vm = PreviewViewModel(screenshot: screenshot) + vm.selectAnnotation(at: 0) + vm.updateSelectedAnnotationBlockSize(20) + guard case .mosaic(let m) = vm.annotations[0] else { + XCTFail("Expected mosaic") + return + } + XCTAssertEqual(m.blockSize, 20) + } + + // MARK: - 5.4 Smart mosaic shares VLM configuration (success criterion 9) + + func testSmartMosaicUsesSharedVLMProviderConfiguration() { + // Local providers need no API key — smart mosaic must accept them as configured + XCTAssertFalse(VLMProviderType.ollama.requiresAPIKey) + XCTAssertTrue(VLMProviderType.openai.requiresAPIKey) + + // The privacy prompt must be part of the shared VLM prompt pair system + XCTAssertEqual(VLMPromptPair.privacy.system.isEmpty, false) + XCTAssertTrue(VLMPromptPair.privacy.user.lowercased().contains("sensitive")) + + // Error copy names the VLM settings it shares + let zh = parseStrings("zh-Hans") + let en = parseStrings("en") + XCTAssertTrue(zh["smartMosaic.error.notConfigured"]?.contains("VLM") == true, "Prompt must point at VLM config") + XCTAssertTrue(en["smartMosaic.error.notConfigured"]?.contains("VLM") == true) + + // Localized feedback states from PRD 5.4 + XCTAssertEqual(zh["smartMosaic.detecting"], "正在识别图中隐私...") + XCTAssertEqual(zh["smartMosaic.noneFound"], "未检测到敏感隐私信息") + XCTAssertEqual(zh["smartMosaic.redacted"], "已自动遮挡 %d 处隐私") + XCTAssertEqual(zh["smartMosaic.error.failed"], "隐私识别失败,请检查网络或重试") + } + + // MARK: - 5.5 Embedded results drawer (success criterion 10) + + func testEmbeddedDrawerStringsAndModel() { + let zh = parseStrings("zh-Hans") + // Drawer chrome from PRD 5.5 + XCTAssertEqual(zh["preview.panel.ocrTab"], "文本识别") + XCTAssertEqual(zh["preview.panel.translateTab"], "翻译") + XCTAssertNotNil(zh["preview.panel.rerun"]) // 重新识别 + XCTAssertNotNil(zh["preview.panel.copyAll"]) // 复制全部 + XCTAssertEqual(zh["preview.panel.close"], "关闭") + XCTAssertEqual(zh["preview.panel.recognizing"], "正在识别...") + XCTAssertNotNil(zh["preview.panel.copyLine"]) // 单行复制 + XCTAssertNotNil(zh["preview.panel.original"]) // 原文 + XCTAssertNotNil(zh["preview.panel.translation"]) // 译文 + + // Model state machine: loading → lines, tab switching keeps data + let model = InPlaceResultsModel() + XCTAssertFalse(model.isVisible) + model.isVisible = true + model.activeTab = .ocr + model.isLoading = true + model.isLoading = false + model.ocrLines = ["line1", "line2"] + model.activeTab = .translate + XCTAssertEqual(model.ocrLines.count, 2, "Tab switch must not drop OCR results") + } + + /// In-place drawer actions must not be routed to the delegate (no external window) + func testInPlaceDrawerActionsAreLocal() { + // The overlay handles .ocr/.translate itself; dispatchAction treats + // .smartMosaic as local-only. Verify the action enum still carries them. + let actions: [SelectionInPlaceToolbar.Action] = [.ocr, .translate, .smartMosaic] + XCTAssertTrue(actions.contains(.ocr)) + XCTAssertTrue(actions.contains(.translate)) + XCTAssertTrue(actions.contains(.smartMosaic)) + } + + // MARK: - Export rendering (fill color + multi-line text) + + func testExportRenderingAppliesFillAndWrapsText() { + let size = 200 + guard let ctx = CGContext( + data: nil, width: size, height: size, bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + XCTFail("context") + return + } + ctx.setFillColor(NSColor.white.cgColor) + ctx.fill(CGRect(x: 0, y: 0, width: size, height: size)) + + let fill = CodableColor(red: 0, green: 0, blue: 1, opacity: 1) // pure blue fill + let rectAnno = RectangleAnnotation( + rect: CGRect(x: 20, y: 20, width: 60, height: 40), + style: StrokeStyle(color: .red, lineWidth: 2), + isFilled: true, + fillColor: fill + ) + ImageExporter.shared.renderAnnotation(.rectangle(rectAnno), in: ctx, imageHeight: CGFloat(size)) + + // Multi-line text renders without trapping + let textAnno = TextAnnotation( + position: CGPoint(x: 100, y: 20), + content: "hello world this wraps", + style: TextStyle(color: .black, fontSize: 12, fontName: ".AppleSystemUIFont"), + wrapWidth: 80 + ) + ImageExporter.shared.renderAnnotation(.text(textAnno), in: ctx, imageHeight: CGFloat(size)) + + guard let image = ctx.makeImage() else { + XCTFail("image") + return + } + // Sample a pixel inside the filled rect: expect blue-dominant + let pixel = Self.pixel(in: image, x: 50, y: 50) + XCTAssertNotNil(pixel) + if let pixel { + XCTAssertGreaterThan(pixel.b, pixel.r, "Fill color must paint (blue > red), got \(pixel)") + XCTAssertGreaterThan(pixel.b, 0.5, "Fill should be strongly blue, got \(pixel)") + } + } + + // MARK: - Cross-line selection & line copy (user report 2026-10-23) + + func testResultsTextBlockSupportsCrossLineSelectionAndLineCopy() { + // One continuous attributed block = selection can span lines + let ocr = ResultsTextBuilder.ocrText(lines: ["line one", "line two", "line three"]) + XCTAssertEqual(ocr.string, "line one\nline two\nline three") + + // Line extraction used by click-to-copy + XCTAssertEqual(ResultsTextView.lineString(at: 0, in: ocr.string), "line one") + XCTAssertEqual(ResultsTextView.lineString(at: 10, in: ocr.string), "line two") + XCTAssertEqual(ResultsTextView.lineString(at: 22, in: ocr.string), "line three") + XCTAssertNil(ResultsTextView.lineString(at: 0, in: "")) + + // Translation pairs stay in one block (original + translated styles) + let seg = BilingualSegment( + original: TextSegment(text: "hello", boundingBox: CGRect(x: 0, y: 0, width: 1, height: 1)), + translated: "你好", + sourceLanguage: "en", + targetLanguage: "zh-Hans" + ) + let tr = ResultsTextBuilder.translationText(segments: [seg]) + XCTAssertTrue(tr.string.contains("hello")) + XCTAssertTrue(tr.string.contains("你好")) + } + + /// Toolbar buttons must keep a hit area around the icon so edge clicks + /// don't fall through to the overlay and cancel the selection. + func testInPlaceToolbarExposesToolAndActionButtons() { + XCTAssertTrue(SelectionInPlaceToolbar.AnnotationTool.allCases.contains(.line)) + // Every tool tooltip carries its shortcut — used for a11y + E2E targeting + for tool in SelectionInPlaceToolbar.AnnotationTool.allCases { + XCTAssertFalse(tool.tooltip.isEmpty) + } + } + + // MARK: - Regression tests (user report 2026-10-23) + + /// AppKit's `hitTest(_:)` hands the point in the **superview's** coordinate + /// system. The in-place toolbar previously compared it against its own + /// `bounds`, so real clicks/hovers fell through and tools went dead. + func testInPlaceToolbarHitTestAcceptsSuperviewCoordinates() { + let parent = NSView(frame: NSRect(x: 0, y: 0, width: 1400, height: 900)) + let bar = SelectionInPlaceToolbar(frame: NSRect(x: 400, y: 300, width: 520, height: 42)) + parent.addSubview(bar) + + // A point inside the bar's frame, expressed in the parent's coordinates + let insideInSuperview = NSPoint(x: 450, y: 320) + XCTAssertNotNil( + bar.hitTest(insideInSuperview), + "hitTest must treat its argument as superview coordinates" + ) + + // A point far outside the bar must pass through to the overlay + XCTAssertNil(bar.hitTest(NSPoint(x: 50, y: 50))) + } + + /// The toolbar must report a size that fits every button. The legacy + /// hard-coded 520pt predates the line/smart-mosaic/eyedropper additions + /// and pushed the leftmost buttons (rectangle, ellipse) outside the view, + /// which killed their hover tooltips and clicks. + func testInPlaceToolbarPreferredSizeAccommodatesEveryButton() { + let bar = SelectionInPlaceToolbar(frame: .zero) + let fitted = bar.fittingContentSize + XCTAssertGreaterThan(fitted.width, 0, "SwiftUI content must lay out") + XCTAssertGreaterThanOrEqual( + bar.preferredSize.width + 0.5, + fitted.width, + "preferredSize must never be narrower than the button row it hosts" + ) + // 10 tools + beautify/SM/OCR/pin/save/upload + undo/redo + cancel/confirm + XCTAssertGreaterThanOrEqual( + bar.preferredSize.width, + CGFloat(SelectionInPlaceToolbar.AnnotationTool.allCases.count + 9) * 24, + "every tool and action button needs a click slot" + ) + } + + /// 截图翻译 must differ from 区域截图: it runs translation right after a + /// region is selected instead of waiting for the confirm button. + func testCaptureModesHaveDistinctPostSelectionActions() { + XCTAssertNil( + CaptureSessionController.CaptureMode.region.automaticActionAfterSelection, + "区域截图 must not auto-trigger any analysis" + ) + XCTAssertEqual( + CaptureSessionController.CaptureMode.screenshotTranslation.automaticActionAfterSelection, + .translate, + "截图翻译 must auto-trigger translation after selection" + ) + XCTAssertNil( + CaptureSessionController.CaptureMode.fullScreen.automaticActionAfterSelection + ) + } + + // MARK: - Results drawer regressions (review 2026-10-23) + + /// Long OCR/translation results must overflow the fixed-height drawer + /// viewport and stay scrollable — the text view has to grow with content. + func testResultsTextViewGrowsBeyondViewportForLongContent() { + let scroll = SelectableTextBlock.makeScrollView( + textView: SelectableTextBlock.makeConfiguredTextView() + ) + let view = scroll.documentView as! ResultsTextView + scroll.frame = NSRect(x: 0, y: 0, width: 280, height: 200) + + XCTAssertTrue(view.isVerticallyResizable, "content height must drive the frame") + XCTAssertTrue(view.textContainer?.widthTracksTextView == true, "must wrap at the drawer width") + + let lines = (0..<60).map { "OCR line \($0) with enough words to wrap at the panel width" } + view.textStorage?.setAttributedString(ResultsTextBuilder.ocrText(lines: lines)) + view.fitToViewport(width: 280) + + XCTAssertGreaterThan( + view.frame.height, 200, + "60 wrapped lines must exceed the 200pt viewport, not get clipped" + ) + } + + /// Only a plain single click copies its line — shift+click extends the + /// selection, drags end in one, and a double-click's first release is + /// followed by word selection (none may clobber the pasteboard). + /// (Event-level testing is impossible here: NSTextView.mouseDown runs a + /// tracking loop that waits on the real event queue.) + func testLineCopyDecisionExcludesShiftClickDragsAndDoubleClicks() { + XCTAssertTrue( + ResultsTextView.shouldCopyLine(clickCount: 1, modifierFlags: [], travel: 1), + "plain single click copies the line" + ) + XCTAssertFalse( + ResultsTextView.shouldCopyLine(clickCount: 1, modifierFlags: [.shift], travel: 0), + "shift+click extends the selection" + ) + XCTAssertFalse( + ResultsTextView.shouldCopyLine(clickCount: 2, modifierFlags: [], travel: 0), + "double-click selects a word" + ) + XCTAssertFalse( + ResultsTextView.shouldCopyLine(clickCount: 1, modifierFlags: [], travel: 5), + "drag ends in a selection" + ) + } + + /// The merged drawer must clear the selection's resize-handle hit areas + /// (child views win hit testing) and never enter the toolbar zone below + /// the selection (toolbar top sits at maxY + 8). + func testResultsPanelFrameClearsHandlesAndToolbarZone() { + let bounds = NSRect(x: 0, y: 0, width: 1400, height: 900) + + // Typical short selection on the right half + let sel = NSRect(x: 600, y: 100, width: 200, height: 120) + let frame = SelectionOverlayView.resultsPanelFrame( + selectionRect: sel, bounds: bounds, handleSize: 8 + ) + XCTAssertGreaterThanOrEqual( + frame.minX - sel.maxX, 10, + "gap must clear the handle hit radius (handleSize + 2)" + ) + XCTAssertLessThanOrEqual( + frame.maxY, sel.maxY + 8 + 0.5, + "drawer bottom must not enter the toolbar zone (sel.maxY + 8)" + ) + + // Right edge too tight: flip to the left without covering the selection + let rightHugging = NSRect(x: 1150, y: 100, width: 220, height: 120) + let flipped = SelectionOverlayView.resultsPanelFrame( + selectionRect: rightHugging, bounds: bounds, handleSize: 8 + ) + XCTAssertLessThanOrEqual( + flipped.maxX, rightHugging.minX + 0.5, + "left-side fallback must not overlap the selection content" + ) + + // Near-fullscreen selection: neither side fits — hug the screen edge + let fullscreenish = NSRect(x: 0, y: 0, width: 1390, height: 890) + let edge = SelectionOverlayView.resultsPanelFrame( + selectionRect: fullscreenish, bounds: bounds, handleSize: 8 + ) + XCTAssertLessThanOrEqual( + edge.maxX, bounds.maxX - 8 + 0.5, + "fallback must stay inside the screen" + ) + } + + /// TextSegment 的坐标原点为左上角,译文必须贴在原文下边缘。 + func testBilingualOverlayAnchorPlacesTranslationBelowTopLeftBox() { + let display = CGSize(width: 300, height: 200) + let topBox = CGRect(x: 0.1, y: 0.1, width: 0.5, height: 0.1) + let top = BilingualOverlayLayout.translationAnchor(for: topBox, in: display) + XCTAssertEqual(top.x, 30, accuracy: 0.5) + XCTAssertEqual(top.y, 41, accuracy: 0.5) + XCTAssertEqual(top.maxWidth, 150, accuracy: 0.5) + + let lowerBox = CGRect(x: 0.25, y: 0.7, width: 0.2, height: 0.1) + let lower = BilingualOverlayLayout.translationAnchor(for: lowerBox, in: display) + XCTAssertEqual(lower.x, 75, accuracy: 0.5) + XCTAssertEqual(lower.y, 161, accuracy: 0.5) + XCTAssertGreaterThan(lower.y, top.y) + + let tiny = BilingualOverlayLayout.translationAnchor( + for: CGRect(x: 0.95, y: 0.5, width: 0.01, height: 0.01), in: display + ) + XCTAssertGreaterThanOrEqual(tiny.fontSize, 10) + XCTAssertLessThanOrEqual(tiny.x + tiny.maxWidth, display.width) + } + + /// VLM gateways forwarding the OpenAI Responses API put a `reasoning` + /// output item in front of the `message` item — parsing must skip it and + /// read the message's text blocks (user report 2026-10-23: deepseek relay). + func testVLMResponsesAPIWithLeadingReasoningItemParses() throws { + let provider = OpenAIVLMProvider( + configuration: try VLMProviderConfiguration( + apiKey: "test", + baseURLString: "https://example.invalid/v1", + modelName: "test" + ) + ) + let payload = """ + {"created_at":1792742674,"error":null,"id":"resp_1e895847","incomplete_details":null, + "model":"deepseek-v4.1-flash-free","object":"response", + "output":[ + {"id":"rs_18e119","type":"reasoning","summary":[]}, + {"id":"msg_01","type":"message","role":"assistant", + "content":[{"type":"output_text","text":"TRANSLATED"}]} + ], + "status":"completed"} + """ + let (content, isTruncated, finishReason) = try OpenAIVLMProvider.extractContentAndStatus(from: Data(payload.utf8)) + XCTAssertEqual(content, "TRANSLATED", "message text must be read past the reasoning item") + XCTAssertFalse(isTruncated) + XCTAssertNil(finishReason) + + let incomplete = payload.replacingOccurrences(of: "\"status\":\"completed\"", with: "\"status\":\"incomplete\"") + let (_, truncated, reason) = try OpenAIVLMProvider.extractContentAndStatus(from: Data(incomplete.utf8)) + XCTAssertTrue(truncated) + XCTAssertEqual(reason, "length") + } + + /// 验证真实选区窗口的绘制结果,不依赖右侧面板或坐标函数自报成功。 + func testTranslationStripsRenderInsideSelectionAndKeepSourcePixels() throws { + let screen = try XCTUnwrap(NSScreen.main) + let bounds = CGRect(origin: .zero, size: screen.frame.size) + let displayID = CGMainDisplayID() + let image = Self.makeImage(width: Int(bounds.width), height: Int(bounds.height)) + let descriptor = DisplayDescriptor( + displayID: displayID, localizedName: "Test", frame: screen.frame, + scaleFactor: 1, isPrimary: true + ) + let snapshot = DisplaySnapshot( + displayID: displayID, descriptor: descriptor, + cocoaBounds: CocoaGlobalRect(screen.frame), quartzBounds: QuartzGlobalRect(bounds), + scaleFactor: 1, image: image + ) + let snapshots = FrozenSnapshotSet(snapshots: [snapshot], candidateWindows: [], triggerReceivedAt: .now) + let window = SelectionOverlayWindow(screen: screen, snapshot: snapshot, snapshotSet: snapshots) + window.isReleasedWhenClosed = false + defer { window.orderOut(nil); window.close() } + let overlay = try XCTUnwrap(window.overlayView) + let selection = CGRect(x: 100, y: 150, width: 600, height: 400) + func mouse(_ type: NSEvent.EventType, _ point: CGPoint) throws -> NSEvent { + try XCTUnwrap(NSEvent.mouseEvent( + with: type, location: overlay.convert(point, to: nil), modifierFlags: [], + timestamp: 0, windowNumber: window.windowNumber, context: nil, + eventNumber: 0, clickCount: 1, pressure: 0 + )) + } + let start = selection.origin + let end = CGPoint(x: selection.maxX, y: selection.maxY) + overlay.mouseDown(with: try mouse(.leftMouseDown, start)) + overlay.mouseDragged(with: try mouse(.leftMouseDragged, end)) + overlay.mouseUp(with: try mouse(.leftMouseUp, end)) + overlay.resultsModel.isVisible = true + overlay.resultsModel.activeTab = .translate + overlay.resultsModel.segments = [BilingualSegment( + original: TextSegment(text: "Hello world", boundingBox: CGRect(x: 0.1, y: 0.2, width: 0.5, height: 0.05)), + translated: "你好,世界", sourceLanguage: "en", targetLanguage: "zh-Hans" + )] + window.orderFront(nil) + let container = try XCTUnwrap(window.contentView) + container.displayIfNeeded() + let bitmap = try XCTUnwrap(container.bitmapImageRepForCachingDisplay(in: container.bounds)) + container.cacheDisplay(in: container.bounds, to: bitmap) + let rendered = try XCTUnwrap(bitmap.cgImage) + let scale = CGFloat(rendered.width) / bounds.width + func color(at point: CGPoint) throws -> (r: Double, g: Double, b: Double) { + try XCTUnwrap(Self.pixel(in: rendered, x: Int(point.x * scale), y: Int(point.y * scale))) + } + let strip = try color(at: CGPoint(x: 162, y: 255)) + XCTAssertLessThan(strip.r, 0.5, "译文背景必须画在选区中的原文下方") + let source = try color(at: CGPoint(x: 162, y: 235)) + XCTAssertGreaterThan(source.r, 0.95, "译文不能覆盖原文区域") + let unused = try color(at: CGPoint(x: 162, y: 475)) + XCTAssertGreaterThan(unused.r, 0.95, "不能沿错误的 Y 轴绘制译文") + let attachment = XCTAttachment(image: NSImage(cgImage: rendered, size: bounds.size)) + attachment.name = "selection-bilingual-overlay" + attachment.lifetime = .keepAlways + add(attachment) + } + + /// Responses API 状态矩阵:截断判定按 status + incomplete_details.reason, +/// 顶层 output_text 与 message 共用同一套状态判定,空输出报明确错误。 + func testVLMResponsesAPIStatusMatrixAndChatRegression() throws { + let provider = try Self.makeOpenAIProvider() + let messageItem = #"{"id":"m1","type":"message","content":[{"type":"output_text","text":"HI"}]}"# + + // incomplete + max_output_tokens:判定截断,进入续写 + let truncatedPayload = """ + {"object":"response","status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"output":[\(messageItem)]} + """ + let (t1, truncated1, reason1) = try OpenAIVLMProvider.extractContentAndStatus(from: Data(truncatedPayload.utf8)) + XCTAssertEqual(t1, "HI") + XCTAssertTrue(truncated1) + XCTAssertEqual(reason1, "length") + + // incomplete + content_filter:报错,不进入续写 + let filteredPayload = """ + {"object":"response","status":"incomplete","incomplete_details":{"reason":"content_filter"},"output":[\(messageItem)]} + """ + XCTAssertThrowsError(try OpenAIVLMProvider.extractContentAndStatus(from: Data(filteredPayload.utf8))) { error in + XCTAssertTrue("\(error)".contains("content_filter"), "应报出具体原因,实际: \(error)") + } + + // 顶层 output_text 与 message 同样受状态判定约束 + let outputTextTruncated = """ + {"object":"response","status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"output_text":"HI"} + """ + let (_, truncated2, _) = try OpenAIVLMProvider.extractContentAndStatus(from: Data(outputTextTruncated.utf8)) + XCTAssertTrue(truncated2, "output_text 分支不能绕过截断状态") + + // 空输出 / 仅 reasoning:在 Responses 分支直接报错,不跌入 Chat 路径 + for payload in [ + #"{"object":"response","status":"completed","output":[]}"#, + #"{"object":"response","status":"completed","output":[{"id":"r1","type":"reasoning","summary":[]}]}"# + ] { + XCTAssertThrowsError(try OpenAIVLMProvider.extractContentAndStatus(from: Data(payload.utf8))) { error in + XCTAssertTrue("\(error)".contains("no message text"), "空输出应给出来源明确的错误,实际: \(error)") + } + } + + // Chat 分支回归:stop 正常、length 截断 + let chat = { (finish: String) in Data(#"{"choices":[{"message":{"content":"HELLO"},"finish_reason":"\#(finish)"}]}"#.utf8) } + let (c1, _, r1) = try OpenAIVLMProvider.extractContentAndStatus(from: chat("stop")) + XCTAssertEqual(c1, "HELLO") + XCTAssertEqual(r1, "stop", "finish_reason 原样透传,截断与否由 isTruncated 表达") + let (_, truncated3, r3) = try OpenAIVLMProvider.extractContentAndStatus(from: chat("length")) + XCTAssertTrue(truncated3) + XCTAssertEqual(r3, "length") + + // 未知格式错误不含原始响应正文 + XCTAssertThrowsError( + try OpenAIVLMProvider.extractContentAndStatus(from: Data("oops".utf8)) + ) { error in + XCTAssertFalse("\(error)".contains("oops"), "错误信息不得回显响应正文") + } + } + + private static func makeOpenAIProvider() throws -> OpenAIVLMProvider { + try OpenAIVLMProvider( + configuration: VLMProviderConfiguration( + apiKey: "test", + baseURLString: "https://example.invalid/v1", + modelName: "test" + ) + ) + } + + // MARK: - Utilities + + private static func makeImage(width: Int, height: Int) -> CGImage { + let ctx = CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + )! + ctx.setFillColor(NSColor.white.cgColor) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: height)) + return ctx.makeImage()! + } + + /// Reads one pixel (top-left origin) from a bitmap-backed CGImage. + private static func pixel(in image: CGImage, x: Int, y: Int) -> (r: Double, g: Double, b: Double)? { + guard let data = image.dataProvider?.data else { return nil } + let bytes = CFDataGetBytePtr(data)! + let bpr = image.bytesPerRow + let bpp = image.bitsPerPixel / 8 + let offset = y * bpr + x * bpp + // premultipliedLast bitmap stores components as R, G, B, A + let r = Double(bytes[offset]) / 255.0 + let g = Double(bytes[offset + 1]) / 255.0 + let b = Double(bytes[offset + 2]) / 255.0 + return (r, g, b) + } +} diff --git a/TransFrameTests/TransFrameErrorTests.swift b/TransFrameTests/TransFrameErrorTests.swift index d9789f2..d5bf881 100644 --- a/TransFrameTests/TransFrameErrorTests.swift +++ b/TransFrameTests/TransFrameErrorTests.swift @@ -91,13 +91,6 @@ final class TransFrameErrorTests: XCTestCase { // MARK: - OCR Errors - func testOCROperationInProgressErrorDescription() { - let error = TransFrameError.ocrOperationInProgress - - XCTAssertNotNil(error.errorDescription) - XCTAssertNotNil(error.recoverySuggestion) - } - func testOCRInvalidImageErrorDescription() { let error = TransFrameError.ocrInvalidImage diff --git a/create_dmg.sh b/create_dmg.sh index a62e8e9..d3dc2bf 100755 --- a/create_dmg.sh +++ b/create_dmg.sh @@ -16,9 +16,10 @@ rm -rf "$PROJECT_PATH/build_artifacts/dmg" rm -rf "$DMG_TEMP_DIR" rm -f "$DMG_PATH" -# 2. 编译 Release 版本 (使用默认/当前架构) -echo "🏗️ 正在编译 Release 版本..." -xcodebuild -project "$XCODE_PROJ" -scheme TransFrame -configuration Release -derivedDataPath "$PROJECT_PATH/build_artifacts/dmg" -quiet +# 2. 编译 Release 版本 (默认 arm64,可传入参数: ./create_dmg.sh x86_64) +TARGET_ARCH="${1:-arm64}" +echo "🏗️ 正在编译 Release 版本 ($TARGET_ARCH)..." +xcodebuild -project "$XCODE_PROJ" -scheme TransFrame -configuration Release -arch "$TARGET_ARCH" -derivedDataPath "$PROJECT_PATH/build_artifacts/dmg" -quiet # 3. 创建临时文件夹结构 echo "📁 准备 DMG 目录结构..." @@ -46,6 +47,50 @@ fi echo "Found app at: $APP_PATH" cp -R "$APP_PATH" "$DMG_TEMP_DIR/" +# 瘦身 Sparkle.framework (按单架构移除冗余切片并 strip) +# 注意:lipo/strip 会使二进制原有签名失效,下方签名步骤会对这些二进制逐层重签。 +SPARKLE_DIR="$DMG_TEMP_DIR/TransFrame.app/Contents/Frameworks/Sparkle.framework" +if [ -d "$SPARKLE_DIR" ]; then + if [ "$TARGET_ARCH" = "arm64" ] || [ "$TARGET_ARCH" = "x86_64" ]; then + echo "⚡ 正在精简 Sparkle.framework 架构 ($TARGET_ARCH)..." + for b in "Versions/B/Sparkle" "Versions/B/Autoupdate" "Versions/B/Updater.app/Contents/MacOS/Updater" "Versions/B/XPCServices/Downloader.xpc/Contents/MacOS/Downloader" "Versions/B/XPCServices/Installer.xpc/Contents/MacOS/Installer"; do + if [ -f "$SPARKLE_DIR/$b" ]; then + if lipo -thin "$TARGET_ARCH" "$SPARKLE_DIR/$b" -output "$SPARKLE_DIR/$b.thin" 2>/dev/null; then + mv "$SPARKLE_DIR/$b.thin" "$SPARKLE_DIR/$b" + else + echo "⚠️ lipo -thin 失败(保留原二进制): $b" >&2 + fi + strip -x "$SPARKLE_DIR/$b" 2>/dev/null || echo "⚠️ strip 失败: $b" >&2 + fi + done + fi +fi + +# 优先使用本机钥匙串中的 Developer ID Application 证书签名(与 GitHub Release 保持一致) +SIGN_IDENTITY=$(security find-identity -v -p codesigning | grep -m1 "Developer ID Application" | awk -F '"' '{print $2}' || true) +APP_BUNDLE="$DMG_TEMP_DIR/TransFrame.app" +if [ -n "$SIGN_IDENTITY" ]; then + echo "🔐 使用 Developer ID 签名: $SIGN_IDENTITY" + # --deep 无法按各自 entitlements 签署嵌套 helper(Sparkle 2 官方明确不支持), + # 且 --timestamp=none 的签名无法通过 Apple 公证;改为从内到外逐层签名 + 安全时间戳。 + CODESIGN_ARGS=(--force --options runtime --timestamp --sign "$SIGN_IDENTITY") +else + echo "⚠️ 未找到 Developer ID 证书,回退为 Ad-hoc 签名" + CODESIGN_ARGS=(--force --sign -) +fi +# 嵌套代码先行(XPC services、Updater.app、Autoupdate),再签框架,最后签 app 外层 +for n in "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework/Versions/B/XPCServices/Downloader.xpc" \ + "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework/Versions/B/XPCServices/Installer.xpc" \ + "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework/Versions/B/Updater.app" \ + "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework/Versions/B/Autoupdate"; do + if [ -e "$n" ]; then + codesign "${CODESIGN_ARGS[@]}" "$n" + fi +done +[ -d "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework" ] && \ + codesign "${CODESIGN_ARGS[@]}" "$APP_BUNDLE/Contents/Frameworks/Sparkle.framework" +codesign "${CODESIGN_ARGS[@]}" "$APP_BUNDLE" + # 创建 Applications 软链接 ln -s /Applications "$DMG_TEMP_DIR/Applications" diff --git a/scripts/e2e/eventsynth.swift b/scripts/e2e/eventsynth.swift new file mode 100644 index 0000000..e5bba22 --- /dev/null +++ b/scripts/e2e/eventsynth.swift @@ -0,0 +1,212 @@ +import Foundation +import CoreGraphics +import AppKit + +// E2E input synthesis + deterministic pixel assertions for TransFrame. +// Usage: +// eventsynth drag x1 y1 x2 y2 [cmd|shift] +// eventsynth click x y [double] +// eventsynth type "text" +// eventsynth key escape|return|tab +// eventsynth shot x y w h out.png +// eventsynth redbbox in.png -> "x y w h" of red-ish pixels +// eventsynth diff a.png b.png -> changed fraction 0...1 + +func mouse(_ type: CGEventType, _ x: Double, _ y: Double, flags: CGEventFlags = [], button: CGMouseButton = .left) { + guard let ev = CGEvent(mouseEventSource: nil, mouseType: type, + mouseCursorPosition: CGPoint(x: x, y: y), mouseButton: button) else { + fputs("CGEvent create failed: \(type) (\(x), \(y))\n", stderr) + exit(1) + } + ev.flags = flags + ev.post(tap: .cghidEventTap) + usleep(20_000) +} + +let args = CommandLine.arguments +guard args.count >= 2 else { fputs("usage: eventsynth \n", stderr); exit(2) } + +func needArgs(_ n: Int, _ usage: String) { + guard args.count >= n else { fputs("usage: \(usage)\n", stderr); exit(2) } +} + +func coord(_ s: String, _ name: String) -> Double { + guard let v = Double(s) else { fputs("bad \(name): \(s)\n", stderr); exit(2) } + return v +} + +// Byte offsets of R/G/B for common 32bpp pixel layouts; nil if unsupported. +// PNGs decoded via CG are usually RGBA big-endian, but don't assume it — +// a wrong layout silently mis-reads colors and breaks pixel assertions. +func rgbOffsets(_ cg: CGImage) -> (r: Int, g: Int, b: Int)? { + guard cg.bitsPerPixel == 32 else { return nil } + let little = cg.byteOrderInfo == .order32Little + switch (cg.alphaInfo, little) { + case (.premultipliedLast, false), (.last, false), (.noneSkipLast, false): + return (0, 1, 2) // RGBA + case (.premultipliedLast, true), (.last, true), (.noneSkipLast, true): + return (3, 2, 1) // memory A,B,G,R (little-endian RGBA) + case (.premultipliedFirst, false), (.first, false), (.noneSkipFirst, false): + return (1, 2, 3) // ARGB + case (.premultipliedFirst, true), (.first, true), (.noneSkipFirst, true): + return (2, 1, 0) // BGRA + default: + return nil + } +} + +switch args[1] { +case "drag": + needArgs(6, "eventsynth drag x1 y1 x2 y2 [cmd|shift]") + let x1 = coord(args[2], "x1"), y1 = coord(args[3], "y1"), x2 = coord(args[4], "x2"), y2 = coord(args[5], "y2") + var flags: CGEventFlags = [] + if args.count > 6 { + if args[6].contains("cmd") { flags.insert(.maskCommand) } + if args[6].contains("shift") { flags.insert(.maskShift) } + } + mouse(.mouseMoved, x1, y1, flags: flags) + usleep(30_000) + mouse(.leftMouseDown, x1, y1, flags: flags) + usleep(30_000) + let steps = 12 + for i in 1...steps { + let t = Double(i) / Double(steps) + mouse(.leftMouseDragged, x1 + (x2 - x1) * t, y1 + (y2 - y1) * t, flags: flags) + } + usleep(30_000) + mouse(.leftMouseUp, x2, y2, flags: flags) + +case "click": + needArgs(4, "eventsynth click x y [double]") + let x = coord(args[2], "x"), y = coord(args[3], "y") + mouse(.mouseMoved, x, y) + mouse(.leftMouseDown, x, y) + mouse(.leftMouseUp, x, y) + if args.count > 4 && args[4] == "double" { + usleep(60_000) + mouse(.leftMouseDown, x, y) + mouse(.leftMouseUp, x, y) + } + +case "type": + needArgs(3, "eventsynth type \"text\"") + let text = args[2] + for ch in text { + let utf = Array(String(ch).utf16) + for down in [true, false] { + if let ev = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: down) { + utf.withUnsafeBufferPointer { buf in + ev.keyboardSetUnicodeString(stringLength: buf.count, unicodeString: buf.baseAddress!) + } + ev.post(tap: .cghidEventTap) + usleep(8_000) + } + } + } + +case "move": + // Hover without clicking — used to verify tooltips + needArgs(4, "eventsynth move x y") + let mx = coord(args[2], "x"), my = coord(args[3], "y") + mouse(.mouseMoved, mx, my) + +case "copy": + // Cmd+C to copy the current selection + for down in [true, false] { + if let ev = CGEvent(keyboardEventSource: nil, virtualKey: 8, keyDown: down) { + ev.flags = .maskCommand + ev.post(tap: .cghidEventTap) + usleep(20_000) + } + } + +case "key": + needArgs(3, "eventsynth key escape|return|tab|space|delete") + let map: [String: CGKeyCode] = ["escape": 53, "return": 36, "tab": 48, "space": 49, "delete": 51] + guard let code = map[args[2]] else { fputs("unknown key\n", stderr); exit(2) } + for down in [true, false] { + if let ev = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: down) { + ev.post(tap: .cghidEventTap) + usleep(15_000) + } + } + +case "shot": + // Delegate to the system screencapture tool (point coordinates, rect crop) + needArgs(7, "eventsynth shot x y w h out.png") + let x = args[2], y = args[3], w = args[4], h = args[5] + let out = args[6] + let p = Process() + p.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture") + p.arguments = ["-x", "-R", "\(x),\(y),\(w),\(h)", out] + do { + try p.run() + } catch { + fputs("screencapture failed to launch: \(error)\n", stderr) + exit(1) + } + p.waitUntilExit() + exit(p.terminationStatus) + +case "redbbox": + // Find the tight bbox of strongly red pixels (default annotation stroke) + needArgs(3, "eventsynth redbbox in.png") + let url = URL(fileURLWithPath: args[2]) + guard let src = NSImage(contentsOf: url), + let cg = src.cgImage(forProposedRect: nil, context: nil, hints: nil), + let data = cg.dataProvider?.data else { exit(1) } + guard let off = rgbOffsets(cg) else { + fputs("unsupported pixel format (bpp=\(cg.bitsPerPixel) alpha=\(cg.alphaInfo.rawValue) order=\(cg.byteOrderInfo.rawValue))\n", stderr) + exit(1) + } + let bytes = CFDataGetBytePtr(data)! + let w = cg.width, h = cg.height, bpr = cg.bytesPerRow, bpp = cg.bitsPerPixel / 8 + var minX = w, minY = h, maxX = -1, maxY = -1, count = 0 + for y in 0.. 140 && r > g + 40 && r > b + 40 { + minX = min(minX, x); minY = min(minY, y) + maxX = max(maxX, x); maxY = max(maxY, y) + count += 1 + } + } + } + if maxX < 0 { + print("NONE") + } else { + print("\(minX) \(minY) \(maxX - minX + 1) \(maxY - minY + 1) \(count)") + } + +case "diff": + needArgs(4, "eventsynth diff a.png b.png") + guard let a = NSImage(contentsOf: URL(fileURLWithPath: args[2])), + let b = NSImage(contentsOf: URL(fileURLWithPath: args[3])), + let ca = a.cgImage(forProposedRect: nil, context: nil, hints: nil), + let cb = b.cgImage(forProposedRect: nil, context: nil, hints: nil), + let da = ca.dataProvider?.data, let db = cb.dataProvider?.data else { exit(1) } + guard let offA = rgbOffsets(ca), let offB = rgbOffsets(cb) else { + fputs("unsupported pixel format\n", stderr) + exit(1) + } + let ba = CFDataGetBytePtr(da)!, bb = CFDataGetBytePtr(db)! + let w = min(ca.width, cb.width), h = min(ca.height, cb.height) + let ra = ca.bytesPerRow, rb = cb.bytesPerRow, pa = ca.bitsPerPixel / 8, pb = cb.bitsPerPixel / 8 + var changed = 0 + for y in 0.. 40 { changed += 1 } + } + } + let frac = Double(changed) / Double(max(1, w * h)) + print(String(format: "%.5f", frac)) + +default: + fputs("unknown command\n", stderr); exit(2) +} diff --git a/scripts/e2e/run_e2e.sh b/scripts/e2e/run_e2e.sh new file mode 100755 index 0000000..3664e2d --- /dev/null +++ b/scripts/e2e/run_e2e.sh @@ -0,0 +1,757 @@ +#!/bin/bash +# TransFrame PRD end-to-end acceptance suite. +# Drives the real installed app via Accessibility + synthesized input, +# asserts outcomes deterministically (window counts, AX titles, pixel bbox). +# Usage: bash scripts/e2e/run_e2e.sh [scenario...] +# scenarios: menu ocr line text square all (default all) +set -u +cd "$(git rev-parse --show-toplevel 2>/dev/null || dirname "$0")" +EV=scripts/e2e/eventsynth +if [ ! -x "$EV" ]; then + echo "构建 eventsynth..." + swiftc -O scripts/e2e/eventsynth.swift -o "$EV" || { echo "eventsynth 构建失败"; exit 1; } +fi +NOTE=/tmp/tf-e2e/note.txt +SHOT=/tmp/tf-e2e +mkdir -p "$SHOT" +WANT_NOTE='TRANSFRAME E2E 424242 +line two content +' +if [ ! -f "$NOTE" ] || [ "$(cat "$NOTE" 2>/dev/null)" != "$WANT_NOTE" ]; then + printf '%s' "$WANT_NOTE" > "$NOTE" +fi + +PASS=0; FAIL=0; FAILED=() +WANT_EXTRA=0 +case "$*" in *all*|*extra*) WANT_EXTRA=1 ;; esac +ok() { PASS=$((PASS+1)); echo " PASS $1"; } +bad() { FAIL=$((FAIL+1)); FAILED+=("$1"); echo " FAIL $1"; } +check(){ if [ "$2" = "$3" ] || { [ "$4" = "contains" ] && [[ "$2" == *"$3"* ]]; }; then ok "$1"; else bad "$1 (got: $2, want: $3)"; fi; } + +# ---------- AX helpers (JXA) ---------- +menu_titles() { + osascript <<'AS' +tell application "System Events" to tell process "TransFrame" + click menu bar item 1 of menu bar 2 + delay 0.35 + set names to name of every menu item of menu 1 of menu bar item 1 of menu bar 2 + key code 53 + return names +end tell +AS +} + +# Wait until System Events can see the TransFrame process +wait_app() { + for _ in $(seq 1 20); do + N=$(osascript -l JavaScript 2>/dev/null <<'JXA' +function run() { + try { return String(Application("System Events").applicationProcesses.byName("TransFrame").windows().length >= 0); } + catch (e) { return "false"; } +} +JXA +) + [ "$N" = "true" ] && return 0 + sleep 0.5 + done + return 1 +} + +trigger_capture() { + osascript >/dev/null <<'AS' +tell application "System Events" to tell process "TransFrame" + click menu bar item 1 of menu bar 2 + delay 0.35 + click menu item "区域截图" of menu 1 of menu bar item 1 of menu bar 2 + return "" +end tell +AS + sleep 1.2 +} + +count_windows() { + osascript -l JavaScript <<'JXA' +function run() { + return Application("System Events").applicationProcesses.byName("TransFrame").windows().length; +} +JXA +} + +overlay_buttons() { + osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + const out = []; + for (const win of proc.windows()) { + for (const el of win.entireContents()) { + try { + const role = el.role(); + if (role === "AXButton" || role === "AXRadioButton") { + const n = String(el.name() || ""); + const d = String(el.description() || ""); + out.push((n && n !== "null" ? n : "") + "||" + d); + } + } catch (e) {} + } + } + return out.join("\n"); +} +JXA +} + +press_button() { + # $1 = AXIdentifier/name substring to match; $2 = optional max Y filter (float) + osascript -l JavaScript - "$1" "${2:-}" <<'JXA' +function run(argv) { + const want = argv[0]; + const yMax = argv[1] ? parseFloat(argv[1]) : Infinity; + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { + const role = el.role(); + if (role === "AXButton" || role === "AXRadioButton") { + let id = ""; + try { for (const a of el.attributes()) { if (a.name() === "AXIdentifier") id = String(a.value()); } } catch (e) {} + const label = id + " " + String(el.name() || "") + " " + String(el.description() || ""); + const matched = want.indexOf("id=") === 0 ? id === want.slice(3) : label.indexOf(want) !== -1; + if (matched) { + let y = -1; + try { y = parseFloat(String(el.position()).split(",")[1]); } catch (e) {} + if (y < yMax) { + try { + for (const a of el.actions()) { + if (String(a.name()) === "AXPress") { + a.perform(); + return "pressed:" + id; + } + } + el.press(); + return "pressed:" + id; + } catch (e) { + return "pressfail:" + e; + } + } + } + } + } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return "notfound:" + want; +} +JXA +} + +# Prints "x y w h" of the button with the given AXIdentifier +button_frame() { + osascript -l JavaScript - "$1" "${2:-}" <<'JXA' +function run(argv) { + const want = argv[0]; + const yMax = argv[1] ? parseFloat(argv[1]) : Infinity; + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { + if (el.role() === "AXButton") { + let id = ""; + try { for (const a of el.attributes()) { if (a.name() === "AXIdentifier") id = String(a.value()); } } catch (e) {} + if (id === want) { + let pos = "0,0", size = "0,0"; + try { for (const a of el.attributes()) { + if (a.name() === "AXPosition") pos = String(a.value()).replace(" ", ""); + if (a.name() === "AXSize") size = String(a.value()).replace(" ", ""); + } } catch (e) {} + const p = pos.split(","); const sz = size.split(","); + const py = parseFloat(p[1]); + if (py < yMax) { + return p[0].replace(",", " ") + " " + p[1] + " " + sz[0].replace(",", " ") + " " + sz[1]; + } + } + } + } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return ""; +} +JXA +} + +# Opens the known note in TextEdit at a fixed spot, raises it above all +# windows and prints its window frame as "x y w h". +setup_textedit() { + open -a TextEdit "$NOTE" 2>/dev/null + sleep 1.5 + # Dismiss save prompts so they never pollute screenshots/OCR + osascript >/dev/null 2>&1 <<'AS' || true +tell application "System Events" to tell process "TextEdit" + repeat with w in windows + repeat with b in buttons of w + if (name of b) is "复原" or (name of b) is "Revert" or (name of b) is "Don\'t Save" then + click b + end if + end repeat + end repeat +end tell +AS + osascript >/dev/null 2>&1 <<'AS' || true +tell application "TextEdit" to activate +delay 0.3 +tell application "System Events" + tell process "TextEdit" + set frontmost to true + if (count of windows) > 0 then + set position of window 1 to {260, 180} + set size of window 1 to {720, 420} + perform action "AXRaise" of window 1 + end if + end tell +end tell +tell application "TextEdit" to activate +AS + sleep 0.8 + local FRAME + FRAME=$(osascript -l JavaScript <<'JXA' 2>/dev/null +function run() { + try { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TextEdit"); + if (proc.windows().length === 0) return "260 180 720 420"; + const w = proc.windows()[0]; + let pos = "260,180", size = "720,420"; + for (const a of w.attributes()) { + if (a.name() === "AXPosition") pos = String(a.value()).replace(/ /g, ""); + if (a.name() === "AXSize") size = String(a.value()).replace(/ /g, ""); + } + return pos.replace(",", " ") + " " + size.replace(",", " "); + } catch (e) { + return "260 180 720 420"; + } +} +JXA +) + echo "${FRAME:-260 180 720 420}" +} + +finish_overlay() { ./scripts/e2e/eventsynth key return; sleep 0.8; } + +# Cancel any leftover overlay so annotations from earlier runs don't leak +reset_overlay() { + wait_app || true + ./scripts/e2e/eventsynth key escape; sleep 0.4 + ./scripts/e2e/eventsynth key escape; sleep 0.4 + osascript -l JavaScript >/dev/null <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + let n = 0; + try { n = proc.windows().length; } catch (e) { return "0"; } + return String(n); +} +JXA + W=$(osascript -l JavaScript <<'JXA' +function run() { + try { return String(Application("System Events").applicationProcesses.byName("TransFrame").windows().length); } + catch (e) { return "0"; } +} +JXA +) + if [ "$W" -gt 1 ]; then + press_button "id=xmark" >/dev/null 2>&1 || true + sleep 0.5 + fi +} + +# ---------- Scenarios ---------- +S=("${@:-all}") +case "${S[0]}" in all) S=(menu ocr line text square) ;; esac + +for sc in "${S[@]}"; do +echo "=== SCENARIO: $sc ===" +reset_overlay +case "$sc" in + +menu) + TITLES=$(menu_titles) + echo " menu: $TITLES" + check "菜单含屏幕取色" "$TITLES" "屏幕取色" contains + check "菜单含打开图片编辑" "$TITLES" "打开图片编辑..." contains + check "菜单含区域截图" "$TITLES" "区域截图" contains + case "$TITLES" in *menu.*) bad "菜单无裸键名 ($TITLES)" ;; *) ok "菜单无裸键名" ;; esac + ;; + +ocr) + # VLM 识别依赖外网;click 1500 200 / shot 320 260 600 260 等固定坐标 + # 需要主屏宽度足够,条件不满足时明确 skip 而非误报 FAIL。 + if ! nc -z -w 3 1.1.1.1 443 2>/dev/null; then + echo " SKIP ocr(无外网连接,VLM 不可达)" + continue + fi + SCR_W=$(osascript -e 'tell application "Finder" to get bounds of window of desktop' 2>/dev/null | awk -F ', ' '{print $3}') + if ! [ "${SCR_W:-0}" -ge 1501 ] 2>/dev/null; then + echo " SKIP ocr(主屏宽度 ${SCR_W:-未知} 不足,固定坐标 1500 会越界)" + continue + fi + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + BEFORE=$(count_windows) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + W1=$(count_windows) + [ "$W1" -gt "$BEFORE" ] && ok "选区 overlay 已出现 ($W1 > $BEFORE)" || bad "选区 overlay 未出现" + R=$(press_button "id=text.viewfinder") + echo " press OCR -> $R" + case "$R" in notfound*) bad "找到 OCR 按钮" ;; *) ok "找到并按下 OCR 按钮" ;; esac + # Remote VLM recognition can take 10-20s; poll until loading text clears. + GOT="" + for _ in $(seq 1 12); do + sleep 2.5 + GOT=$(osascript -l JavaScript 2>/dev/null <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + const out = []; + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { + const role = String(el.role() || ""); + if (role === "AXStaticText" || role === "AXTextArea") { + const v = (String(el.value() || "") + " " + String(el.name() || "")).trim(); + if (v && v !== "null") out.push(v); + } + } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return out.join(" | "); +} +JXA +) + case "$GOT" in *正在识别*) continue ;; esac + break + done + W2=$(count_windows) + [ "$W2" = "$W1" ] && ok "OCR 后窗口数不变(无外部新窗口)" || bad "OCR 弹出了新窗口 ($W1 -> $W2)" + # presence check without pressing: search drawer close (xmark below toolbar row) + FOUND=$(osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { + if (el.role() === "AXButton") { + let id = ""; + try { for (const a of el.attributes()) { if (a.name() === "AXIdentifier") id = String(a.value()); } } catch (e) {} + if (id === "xmark") { + let y = -1; + try { y = parseFloat(String(el.position()).split(",")[1]); } catch (e) {} + if (y < 550) return "drawer-close-present"; + } + } + } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return "drawer-close-absent"; +} +JXA +) + [ "$FOUND" = "drawer-close-present" ] && ok "内嵌面板出现(含关闭按钮)" || bad "内嵌面板未出现 ($FOUND)" + TXT=$(osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + const out = []; + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { + const role = el.role(); + if (role === "AXStaticText" || role === "AXTextArea" || role === "AXTextField") { + const v = String(el.value() || "") + " " + String(el.name() || ""); + if (v.trim() && v.trim() !== "null") out.push(v.trim()); + } + } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return out.join(" | "); +} +JXA +) + echo " texts: $TXT" + echo "$TXT" | grep -q "424242" && ok "面板含 OCR 识别文本 (424242)" || bad "OCR 结果未显示预期文本: $TXT" + R=$(press_button "id=xmark" 550) + case "$R" in notfound*) bad "按下关闭按钮" ;; *) ok "关闭面板" ;; esac + sleep 0.5 + press_button "xmark" 550 | grep -q notfound && ok "面板已收起" || bad "面板未收起" + finish_overlay + ;; + +line) + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + $EV shot 320 260 600 260 "$SHOT/line-before.png" + R=$(press_button "id=line.diagonal") + echo " press line tool -> $R" + case "$R" in notfound*) bad "找到直线工具" ;; *) ok "找到并激活直线工具" ;; esac + $EV drag $((SELX + 80)) $((SELY + 50)) $((SELX + 540)) $((SELY + 210)) + sleep 0.6 + $EV shot 320 260 600 260 "$SHOT/line-after.png" + D=$($EV diff "$SHOT/line-before.png" "$SHOT/line-after.png") + echo " pixel diff: $D" + awk -v d="$D" 'BEGIN{exit !(d > 0.0005)}' && ok "直线绘制产生像素变化" || bad "直线绘制无变化 (diff=$D)" + finish_overlay + ;; + +text) + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + $EV shot 320 260 600 260 "$SHOT/text-before.png" + R=$(press_button "id=textbox") + echo " press text tool -> $R" + case "$R" in notfound*) bad "找到文本工具" ;; *) ok "找到并激活文本工具" ;; esac + $EV click $((SELX + 120)) $((SELY + 70)) + sleep 0.6 + $EV type "E2ETEXT" + sleep 0.3 + $EV click $((SELX + 420)) $((SELY + 250)) + sleep 0.8 + $EV shot 320 260 600 260 "$SHOT/text-after.png" + D=$($EV diff "$SHOT/text-before.png" "$SHOT/text-after.png") + echo " pixel diff after commit: $D" + awk -v d="$D" 'BEGIN{exit !(d > 0.0005)}' && ok "文本提交后画布有内容" || bad "文本未提交 (diff=$D)" + # drag the committed text box and confirm the pixels move + $EV drag $((SELX + 120)) $((SELY + 70)) $((SELX + 240)) $((SELY + 170)) + sleep 0.6 + $EV shot 320 260 600 260 "$SHOT/text-moved.png" + D2=$($EV diff "$SHOT/text-after.png" "$SHOT/text-moved.png") + echo " pixel diff after drag: $D2" + awk -v d="$D2" 'BEGIN{exit !(d > 0.0003)}' && ok "文本框可拖动" || bad "文本框拖动无变化 (diff=$D2)" + finish_overlay + ;; + +square) + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + R=$(press_button "id=rectangle") + echo " press rect tool -> $R" + case "$R" in notfound*) bad "找到矩形工具" ;; *) ok "找到并激活矩形工具" ;; esac + # dismiss any system dialogs (e.g. Keychain prompts) before measuring + # (restrict to known system prompt processes so we never click other apps) + osascript -l JavaScript >/dev/null <<'JXA' +function run() { + const se = Application("System Events"); + const allowed = ["UserNotificationCenter", "SecurityAgent"]; + for (const p of se.applicationProcesses()) { + if (!allowed.includes(String(p.name()))) continue; + try { + for (const w of p.windows()) { + for (const b of w.buttons()) { + const n = String(b.name() || ""); + if (n === "取消" || n === "Cancel") { + try { for (const a of b.actions()) { if (String(a.name()) === "AXPress") a.perform(); } } catch (e) {} + return "dismissed"; + } + } + } + } catch (e) {} + } + return "none"; +} +JXA + $EV drag $((SELX + 340)) $((SELY + 50)) $((SELX + 560)) $((SELY + 180)) cmd + sleep 0.6 + # tight region around the expected draw area (avoid stray UI pixels) + $EV shot 590 270 300 250 "$SHOT/square.png" + BBOX=$($EV redbbox "$SHOT/square.png") + echo " red bbox: $BBOX" + if [ "$BBOX" = "NONE" ]; then + bad "⌘ 绘制未出现红色矩形" + else + set -- $BBOX + W=$3; H=$4 + DIFF=$(( W > H ? W - H : H - W )) + echo " w=$W h=$H diff=$DIFF" + [ "$DIFF" -le 4 ] && ok "⌘ 绘制保持正方形 (|w-h|=$DIFF <= 4)" || bad "⌘ 未锁定正方形 (w=$W h=$H)" + fi + finish_overlay + ;; +esac +done + +# --- Scenario: edge-click (user report 1/3) --- +run_edge_click() { + echo "=== SCENARIO: edge-click ===" + reset_overlay + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1.2 + FRAME=$(button_frame "line.diagonal") + if [ -z "$FRAME" ]; then + bad "读取直线按钮位置" + return + fi + set -- $FRAME + BX=$1; BY=$2; BW=$3; BH=$4 + # click 3pt above the icon visual (old hit area leaked here and cancelled the selection) + EDGE_X=$(( BX + BW / 2 )) + EDGE_Y=$(( BY - 3 )) + # 1) click the TOP EDGE of the line button (1pt inside its frame): must activate + B0=$(osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + let n = 0; + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { if (el.role() === "AXButton" || el.role() === "AXRadioButton") n++; } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return String(n); +} +JXA +) + $EV click $((BX + BW / 2)) $((BY + 1)) + sleep 0.6 + B1=$(osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + let n = 0; + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { if (el.role() === "AXButton" || el.role() === "AXRadioButton") n++; } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return String(n); +} +JXA +) + echo " edge-click y=BY+1: buttons $B0 -> $B1" + [ "$B1" -gt "$B0" ] && ok "按钮边缘点击激活工具" || bad "按钮边缘点击未激活 ($B0->$B1)" + # drawing with the activated tool produces a localized change + BX1=$((SELX + 80)); BY1=$((SELY + 50)); BX2=$((SELX + 540)); BY2=$((SELY + 210)) + $EV shot $((BX1 - 20)) $((BY1 - 20)) $((BX2 - BX1 + 40)) $((BY2 - BY1 + 40)) "$SHOT/edge-before.png" + $EV drag $BX1 $BY1 $BX2 $BY2 + sleep 0.6 + $EV shot $((BX1 - 20)) $((BY1 - 20)) $((BX2 - BX1 + 40)) $((BY2 - BY1 + 40)) "$SHOT/edge-after.png" + D=$($EV diff "$SHOT/edge-before.png" "$SHOT/edge-after.png") + echo " draw diff=$D" + awk -v d="$D" 'BEGIN{exit !(d > 0.0005 && d < 0.3)}' && ok "边缘激活后绘制生效 (diff=$D)" || bad "绘制行为异常 (diff=$D)" + # 2) click just OUTSIDE the button (2pt above): must be inert (no cancel) + $EV shot $((BX - 40)) $((BY - 30)) 200 80 "$SHOT/out-before.png" + $EV click $((BX + BW / 2)) $((BY - 2)) + sleep 0.6 + $EV shot $((BX - 40)) $((BY - 30)) 200 80 "$SHOT/out-after.png" + OD=$($EV diff "$SHOT/out-before.png" "$SHOT/out-after.png") + BC=$(osascript -l JavaScript <<'JXA' +function run() { + const se = Application("System Events"); + const proc = se.applicationProcesses.byName("TransFrame"); + let n = 0; + for (const win of proc.windows()) { + const stack = [win]; + while (stack.length) { + const el = stack.pop(); + try { if (el.role() === "AXButton" || el.role() === "AXRadioButton") n++; } catch (e) {} + try { for (const k of el.uiElements()) stack.push(k); } catch (e) {} + } + } + return String(n); +} +JXA +) + echo " outside-2pt click: buttons=$BC toolbar-area diff=$OD" + [ "$BC" -gt 0 ] && ok "按钮外点击不取消选区(工具栏保持)" || bad "按钮外点击取消了选区 (buttons=$BC)" + finish_overlay +} + +# --- Scenario: outside-click (user report 2/3) --- +run_outside_click() { + echo "=== SCENARIO: outside-click ===" + reset_overlay + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + # plain click far from selection and toolbar + $EV click 1500 200 + sleep 0.8 + R=$(press_button "id=text.viewfinder") + case "$R" in + pressed*) ok "选区外单击后选区与工具栏保持(OCR 按钮仍在)" ;; + *) bad "选区外单击取消了选区 ($R)" ;; + esac + finish_overlay +} + +# --- Scenario: crossline (user report 3/3) --- +run_crossline() { + echo "=== SCENARIO: crossline ===" + reset_overlay + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1 + R=$(press_button "id=text.viewfinder") + case "$R" in notfound*) bad "打开 OCR 面板" ; return ;; esac + sleep 4 + # drawer close button (xmark below toolbar row) anchors the drawer + ANCHOR=$(button_frame "xmark" 550) + # drag-select vertically across several OCR lines inside the text block + if [ -n "$ANCHOR" ]; then + set -- $ANCHOR + # text block lives below the drawer header, left of the close button + TX=$(( $1 - 200 )); TY=$(( $2 + 60 )) + $EV drag $TX $TY $((TX + 90)) $((TY + 150)) + else + DX=$((SELX + SELW + 20)) + $EV drag $((DX + 30)) $((SELY + 40)) $((DX + 120)) $((SELY + 160)) + fi + sleep 0.4 + $EV copy + sleep 0.4 + CLIP=$(pbpaste) + LINES=$(printf '%s' "$CLIP" | wc -l | tr -d ' ') + TOTAL=$((LINES + 1)) + echo " clipboard lines=$TOTAL content=[$CLIP]" + [ "$TOTAL" -ge 2 ] && [ -n "$CLIP" ] && ok "跨行选择复制了多行文本 ($TOTAL 行)" || bad "跨行选择失败 (行数=$TOTAL)" + finish_overlay +} + +# --- Scenario: regressions 2026-10-23 --- +# 1) hover tooltips on rectangle/ellipse tools 2) real clicks activate tools +# 3) 截图翻译 auto-translates; 区域截图 does not +run_regressions() { + echo "=== SCENARIO: regressions ===" + reset_overlay + TF=$(setup_textedit) + [ -n "$TF" ] || TF="260 180 720 420" + set -- $TF + SELX=$(( $1 + 4 )); SELY=$(( $2 + 30 )); SELW=$(( $3 - 8 )); SELH=$(( $4 - 60 )) + + # ---- 3a) 区域截图: selection must NOT open any drawer ---- + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1.2 + R=$(press_button "id=xmark" 550) + case "$R" in + notfound*) ok "区域截图选区后无自动抽屉" ;; + *) bad "区域截图误开了抽屉 ($R)" ;; + esac + finish_overlay + sleep 0.8 + + # ---- tooltip: hover rectangle tool, tooltip must render ---- + trigger_capture + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1.2 + FRAME=$(button_frame "rectangle") + if [ -z "$FRAME" ]; then + bad "读取矩形工具位置" + else + set -- $FRAME + BX=$1; BY=$2; BW=$3; BH=$4 + # baseline without hover + $EV shot $((BX - 30)) $((BY - 50)) $((BW + 60)) $((BH + 30)) "$SHOT/tip-before.png" + $EV move $((BX + BW / 2)) $((BY + BH / 2)) + sleep 0.8 + $EV shot $((BX - 30)) $((BY - 50)) $((BW + 60)) $((BH + 30)) "$SHOT/tip-after.png" + TD=$($EV diff "$SHOT/tip-before.png" "$SHOT/tip-after.png") + echo " hover rect tooltip diff=$TD" + awk -v d="$TD" 'BEGIN{exit !(d > 0.0002)}' && ok "矩形工具悬停出现提示" || bad "矩形工具无悬停提示 (diff=$TD)" + fi + + # ---- real click on ellipse tool activates it (style bar appears) ---- + FRAME2=$(button_frame "circle") + if [ -z "$FRAME2" ]; then + bad "读取椭圆工具位置" + else + set -- $FRAME2 + CX=$1; CY=$2; CW=$3; CH=$4 + $EV click $((CX + CW / 2)) $((CY + CH / 2)) + sleep 0.7 + FRAME3=$(button_frame "circle.fill") + [ -n "$FRAME3" ] && ok "真实点击激活椭圆工具(出现填充切换)" || bad "真实点击无法激活工具" + fi + finish_overlay + sleep 0.8 + + # ---- 3b) 截图翻译: selection must auto-open the translate drawer ---- + osascript >/dev/null 2>&1 <<'AS' +tell application "System Events" to tell process "TransFrame" + click menu bar item 1 of menu bar 2 + delay 0.35 + click menu item "截图翻译" of menu 1 of menu bar item 1 of menu bar 2 +end tell +AS + sleep 1.5 + $EV drag $SELX $SELY $((SELX + SELW)) $((SELY + SELH)) + sleep 1.5 + R2=$(press_button "id=xmark" 550) + case "$R2" in + pressed*) ok "截图翻译选区后自动展开翻译抽屉" ;; + *) bad "截图翻译未自动展开抽屉 ($R2)" ;; + esac + finish_overlay +} + +if [ "$WANT_EXTRA" = "1" ]; then + run_edge_click + run_outside_click + run_crossline + run_regressions +fi + +echo "" +echo "======== E2E RESULT: PASS=$PASS FAIL=$FAIL ========" +if [ "$FAIL" -gt 0 ]; then printf 'failed: %s\n' "${FAILED[@]}"; exit 1; fi