diff --git a/README.md b/README.md index f9d22ee..bdc1702 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,8 @@ findmy people findmy people --json findmy people --no-log -# Click a row and OCR the detail pane (precise address). +# Read one matching person. `--zoom` clicks the row and OCRs the detail pane +# for a street address — see the note on macOS 26+ under Limitations. findmy person "Omar Shahine" findmy person "Omar Shahine" --json @@ -173,6 +174,14 @@ diagnostic — TCC denied is more common than missing display. ## Limitations +- **`--zoom` yields no street address on macOS 26 and later.** macOS 26 + replaced FindMy's split view with a floating sidebar over a full-window map. + A row click no longer opens a detail pane; it opens a callout pinned to the + map carrying the same coarse location and staleness the sidebar already + showed. There is nothing more precise on screen to OCR, so `--zoom` prints a + warning to stderr and leaves `precise_address` unset rather than guessing. + Everything else — `people`, `devices`, `items`, `watch`, `log` — is + unaffected. Tracked in [#13](https://github.com/omarshahine/findmy-cli/issues/13). - **The display must be awake and unlocked.** WindowServer stops compositing when the display sleeps, so `screencapture` returns a 99 KB all-black PNG. The CLI detects this and tells you to wake the keyboard. There is no diff --git a/cmd/findmy/main.go b/cmd/findmy/main.go index d2932c7..26e3ad5 100644 --- a/cmd/findmy/main.go +++ b/cmd/findmy/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "os" @@ -360,7 +361,7 @@ func runDevice(args []string) { os.Exit(1) } detailShot := filepath.Join(tmpDir(), "device-detail.png") - must(enrichWithDetailPane(w, shot, detailShot, nameLine, sidebarRightPx, opts.keep, func(precise, city, region, postal string) { + must(enrichWithDetailPane(w, shot, detailShot, nameLine, sidebarRightPx, opts.keep, match.Name, func(precise, city, region, postal string) { match.PreciseAddress = precise match.City = city match.Region = region @@ -728,7 +729,7 @@ func runPerson(args []string) { os.Exit(1) } detailShot := filepath.Join(tmpDir(), "person-detail.png") - must(enrichWithDetailPane(w, shot, detailShot, nameLine, sidebarRightPx, opts.keep, func(precise, city, region, postal string) { + must(enrichWithDetailPane(w, shot, detailShot, nameLine, sidebarRightPx, opts.keep, match.Name, func(precise, city, region, postal string) { match.PreciseAddress = precise match.City = city match.Region = region @@ -785,7 +786,7 @@ func findSidebarNameLine(lines []findmy.TextLine, sidebarRightPx, textColMinPx i return findmy.TextLine{}, false } -func enrichWithDetailPane(w *findmy.Window, sidebarShotPath, detailShotPath string, clickLine findmy.TextLine, sidebarRightPx int, keep bool, apply func(precise, city, region, postal string)) error { +func enrichWithDetailPane(w *findmy.Window, sidebarShotPath, detailShotPath string, clickLine findmy.TextLine, sidebarRightPx int, keep bool, entityName string, apply func(precise, city, region, postal string)) error { clickX := clickLine.X + clickLine.Width/2 clickY := clickLine.Y + clickLine.Height/2 screenX, screenY := windowPointFromImagePoint(w, sidebarShotPath, clickX, clickY) @@ -804,13 +805,29 @@ func enrichWithDetailPane(w *findmy.Window, sidebarShotPath, detailShotPath stri if err != nil { return err } - precise, city, region, postal := findmy.ExtractDetailPaneAddress(lines, sidebarRightPx) + precise, city, region, postal, err := findmy.ExtractDetailPaneAddress(lines, sidebarRightPx, entityName) + if err != nil && !errors.Is(err, findmy.ErrNoDetailPane) { + return err + } if precise != "" || city != "" || region != "" || postal != "" { apply(precise, city, region, postal) + return nil } + // Not fatal: the coarse sidebar reading is still good, and callers pipe + // --json into scripts that a non-zero exit would break. Warn loudly + // instead — the one thing --zoom must never do is invent an address. + warnNoPreciseAddress(entityName) return nil } +func warnNoPreciseAddress(entityName string) { + fmt.Fprintf(os.Stderr, "warning: --zoom read no precise address for %q.\n", entityName) + fmt.Fprintln(os.Stderr, " macOS 26 replaced FindMy's split view with a floating sidebar over a") + fmt.Fprintln(os.Stderr, " full-window map. Selecting a row now opens a map callout carrying the") + fmt.Fprintln(os.Stderr, " same coarse location as the sidebar, not a street address, so there is") + fmt.Fprintln(os.Stderr, " nothing more precise on screen to read. See issue #13.") +} + func zoomDelay() time.Duration { const fallback = 600 * time.Millisecond if raw := os.Getenv("FINDMY_ZOOM_DELAY_MS"); raw != "" { @@ -823,14 +840,21 @@ func zoomDelay() time.Duration { } // pixelLayout returns the sidebar-right and name-column-left thresholds in -// image pixels. The FindMy sidebar is ~340pt wide; the avatar column is -// ~100pt with the avatar circle centered around 50pt, so an 80pt cutoff -// drops centered avatar OCR fragments while admitting real name/location -// text that begins around 90pt. We use a float scale because some displays -// (e.g. a 4K dummy plug) report non-integer pixel-per-point ratios. +// image pixels. The FindMy sidebar is ~340pt wide; the avatar column holds a +// circle centered around 40pt, so a 60pt cutoff drops centered avatar OCR +// fragments while admitting real name/location text. +// +// 60pt rather than 80pt because macOS 26 replaced the split view with a +// floating sidebar panel inset from the window edge, tightening the avatar +// column: names now start around 66pt from the window's left edge (measured +// on 26.6.2), where an 80pt cutoff would drop every row. The older split-view +// layout put names around 90pt, so 60pt admits both. +// +// We use a float scale because some displays (e.g. a 4K dummy plug) report +// non-integer pixel-per-point ratios. func pixelLayout(w *findmy.Window, imagePath string) (sidebarRightPx, textColMinPx int) { scale := imageScale(w, imagePath) - return int(340 * scale), int(80 * scale) + return int(340 * scale), int(60 * scale) } func windowPointFromImagePoint(w *findmy.Window, imagePath string, px, py int) (int, int) { diff --git a/internal/findmy/detail_pane_test.go b/internal/findmy/detail_pane_test.go index 4988c9c..766e272 100644 --- a/internal/findmy/detail_pane_test.go +++ b/internal/findmy/detail_pane_test.go @@ -1,6 +1,9 @@ package findmy -import "testing" +import ( + "errors" + "testing" +) func TestExtractDetailPaneAddressSplitsUSAddress(t *testing.T) { lines := []TextLine{ @@ -12,8 +15,11 @@ func TestExtractDetailPaneAddressSplitsUSAddress(t *testing.T) { {Text: "Notifications", X: 720, Y: 360, Width: 150, Height: 24}, } - precise, city, region, postal := ExtractDetailPaneAddress(lines, 680) + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "Omar Shahine") + if err != nil { + t.Fatalf("err = %v, want nil", err) + } if precise != "10001 NE 8th St" { t.Fatalf("precise = %q, want %q", precise, "10001 NE 8th St") } @@ -30,8 +36,11 @@ func TestExtractDetailPaneAddressFallsBackForUnsplitAddress(t *testing.T) { {Text: "5 mi away • Updated 2 min ago", X: 730, Y: 220, Width: 300, Height: 24}, } - precise, city, region, postal := ExtractDetailPaneAddress(lines, 680) + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "Sadie Van Horn") + if err != nil { + t.Fatalf("err = %v, want nil", err) + } if precise != "10 Downing Street, London SW1A 2AA" { t.Fatalf("precise = %q, want fallback joined address", precise) } @@ -51,8 +60,11 @@ func TestExtractDetailPaneAddressIgnoresSidebarAndButtons(t *testing.T) { {Text: "Cupertino, CA", X: 760, Y: 250, Width: 180, Height: 24}, } - precise, city, region, postal := ExtractDetailPaneAddress(lines, 680) + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "MacBook Pro") + if err != nil { + t.Fatalf("err = %v, want nil", err) + } if precise != "1 Apple Park Way" { t.Fatalf("precise = %q, want %q", precise, "1 Apple Park Way") } @@ -60,3 +72,129 @@ func TestExtractDetailPaneAddressIgnoresSidebarAndButtons(t *testing.T) { t.Fatalf("split = (%q, %q, %q), want (Cupertino, CA, empty)", city, region, postal) } } + +// TestExtractDetailPaneAddressRejectsMapCanvas pins the macOS 26+ redesign +// (issue #13). The floating sidebar sits over a full-window map, so there is +// no detail pane at all and everything right of the sidebar is map furniture. +// The OCR fixture is a real `findmy person --zoom` capture on macOS 26.6.2, +// which previously produced precise_address = "Champaign Point, 3D". +func TestExtractDetailPaneAddressRejectsMapCanvas(t *testing.T) { + lines := []TextLine{ + {Text: "People", X: 80, Y: 124, Width: 90, Height: 24}, + {Text: "Lora Shahine", X: 133, Y: 324, Width: 169, Height: 27}, + {Text: "Winston-Salem, NC • 3 min. ago", X: 136, Y: 353, Width: 300, Height: 24}, + {Text: "3D", X: 1900, Y: 40, Width: 40, Height: 24}, + {Text: "Champaign Point", X: 748, Y: 345, Width: 200, Height: 24}, + {Text: "Kirkland", X: 1128, Y: 595, Width: 160, Height: 30}, + {Text: "NE 85TH ST", X: 1400, Y: 590, Width: 180, Height: 20}, + {Text: "Lake Washington", X: 700, Y: 1390, Width: 190, Height: 24}, + } + + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "Lora Shahine") + + if !errors.Is(err, ErrNoDetailPane) { + t.Fatalf("err = %v, want ErrNoDetailPane", err) + } + if precise != "" || city != "" || region != "" || postal != "" { + t.Fatalf("got (%q, %q, %q, %q), want all empty — map labels are not an address", precise, city, region, postal) + } +} + +// TestExtractDetailPaneAddressRejectsMapCallout is the second half of issue +// #13. Once the click lands, the redesigned FindMy answers with a callout +// pinned to the map: the entity's name over the same coarse location and +// staleness the sidebar already showed, surrounded by street labels. The +// callout header matches the entity, so header matching alone is not enough — +// the bullet-joined location must be rejected as an address, and the street +// labels must be rejected for sitting outside the header's column. +// +// Fixture is a real `findmy person "Lora Shahine" --zoom` capture on macOS +// 26.6.2, which produced precise_address = +// "Winston-Salem, NC • Now, WAREHAM LN, CHANCELLORSVILLE DR, HAGEN LN". +func TestExtractDetailPaneAddressRejectsMapCallout(t *testing.T) { + lines := []TextLine{ + {Text: "BETHABARA PARK BLVD", X: 950, Y: 379, Width: 260, Height: 20}, + {Text: "Salemtowne", X: 890, Y: 512, Width: 150, Height: 24}, + {Text: "Lora Shahine", X: 1324, Y: 738, Width: 170, Height: 27}, + {Text: "Winston-Salem, NC • Now", X: 1327, Y: 773, Width: 240, Height: 22}, + {Text: "WAREHAM LN", X: 1132, Y: 862, Width: 160, Height: 20}, + {Text: "CHANCELLORSVILLE DR", X: 918, Y: 1056, Width: 280, Height: 20}, + {Text: "HAGEN LN", X: 1904, Y: 1147, Width: 130, Height: 20}, + {Text: "BULL RUN RD", X: 907, Y: 1342, Width: 160, Height: 20}, + } + + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "Lora Shahine") + + if err != nil { + t.Fatalf("err = %v, want nil (the callout header did match)", err) + } + if precise != "" || city != "" || region != "" || postal != "" { + t.Fatalf("got (%q, %q, %q, %q), want all empty — the callout carries no street address", precise, city, region, postal) + } +} + +func TestLooksLikeAddressLine(t *testing.T) { + cases := []struct { + line string + want bool + }{ + {"10001 NE 8th St", true}, + {"1 Apple Park Way", true}, + {"Cupertino, CA", true}, + {"Bellevue, WA 98004", true}, + {"WAREHAM LN", true}, + // Substring matching used to accept these: "Winston" contains "st", + // "Redmond" contains "rd", "Kirkland" contains "ln". + {"Winston-Salem, NC • Now", false}, + {"Kirkland", false}, + {"Champaign Point", false}, + {"Salemtowne", false}, + } + for _, c := range cases { + if got := looksLikeAddressLine(c.line); got != c.want { + t.Errorf("looksLikeAddressLine(%q) = %v, want %v", c.line, got, c.want) + } + } +} + +// A detail pane that is genuinely on screen but has no address (offline +// device) is a different outcome from having no pane at all: no error, and +// nothing to apply. +func TestExtractDetailPaneAddressPaneWithoutAddress(t *testing.T) { + lines := []TextLine{ + {Text: "Bike Shed Keys", X: 760, Y: 110, Width: 220, Height: 30}, + {Text: "No location found", X: 760, Y: 155, Width: 230, Height: 24}, + {Text: "Play Sound", X: 760, Y: 200, Width: 120, Height: 24}, + } + + precise, city, region, postal, err := ExtractDetailPaneAddress(lines, 680, "Bike Shed Keys") + + if err != nil { + t.Fatalf("err = %v, want nil (the pane is present, it just has no address)", err) + } + if precise != "" || city != "" || region != "" || postal != "" { + t.Fatalf("got (%q, %q, %q, %q), want all empty", precise, city, region, postal) + } +} + +func TestMatchesEntityHeader(t *testing.T) { + cases := []struct { + line, name string + want bool + }{ + {"Omar Shahine", "Omar Shahine", true}, + {"omar shahine", "Omar Shahine", true}, + {"Omar Shahine…", "Omar Shahine", true}, + {"Omar Sunshine", "Omar Shahine", true}, // one OCR-mangled word of two + {"Champaign Point", "Lora Shahine", false}, + {"3D", "Lora Shahine", false}, + {"Kirkland", "Omar's iPhone", false}, + {"", "Omar Shahine", false}, + {"Omar Shahine", "", false}, + } + for _, c := range cases { + if got := matchesEntityHeader(c.line, c.name); got != c.want { + t.Errorf("matchesEntityHeader(%q, %q) = %v, want %v", c.line, c.name, got, c.want) + } + } +} diff --git a/internal/findmy/findmy.go b/internal/findmy/findmy.go index 7094eac..1d0bb6a 100644 --- a/internal/findmy/findmy.go +++ b/internal/findmy/findmy.go @@ -2,6 +2,7 @@ package findmy import ( "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -169,12 +170,22 @@ func MainWindow() (*Window, error) { // those coordinates and pollute the OCR with terminal/desktop content when // FindMy isn't strictly frontmost. // +// `-o` omits the window's drop shadow. Without it screencapture pads the +// bitmap with ~34pt of shadow on every side, so the image is larger than the +// window (e.g. 2184x1672 for a 1024x768 window on a 2x display). Every caller +// that converts between image pixels and window points — the sidebar column +// thresholds and the click mapping — assumes image (0,0) is window (0,0) and +// that width ratio is the backing scale. The shadow breaks both assumptions: +// it inflates the derived scale to 2.13 and offsets every mapped point by the +// shadow inset. With `-o` the bitmap is exactly window size times backing +// scale, and the arithmetic is exact. +// // Capture fails with a friendly error when the display is asleep or the // window's backing store hasn't been populated yet (Catalyst quirk after // rapid focus changes). Both produce "could not create image from window" // or a tiny all-black PNG. func Capture(w *Window, dest string) error { - cmd := exec.Command("/usr/sbin/screencapture", "-x", "-l", fmt.Sprintf("%d", w.WindowID), "-t", "png", dest) + cmd := exec.Command("/usr/sbin/screencapture", "-x", "-o", "-l", fmt.Sprintf("%d", w.WindowID), "-t", "png", dest) cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return diagnoseCaptureFailure(err) @@ -668,11 +679,46 @@ func isBattery(s string) bool { var cityRegionPostalRE = regexp.MustCompile(`^([A-Za-z .'-]+),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?$`) +// detailPaneColumnTolerancePx is how far an address line's left edge may sit +// from the header's before it stops being part of the same pane. Vision +// reports left edges within a couple of pixels for genuinely left-aligned +// text; 40px leaves room for a 2x display's rounding without admitting a +// neighbouring map label. +const detailPaneColumnTolerancePx = 40 + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +// ErrNoDetailPane reports that the right-hand region of the window is not a +// detail pane. macOS 26 replaced FindMy's split view with a floating sidebar +// over a full-bleed map, so a row click no longer reveals a pane containing +// the selected entity's street address; the region to the right of the +// sidebar is map canvas. +// +// This must be a hard error rather than a silent empty result. Before the +// guard existed, ExtractDetailPaneAddress happily OCR'd whatever sat right of +// the sidebar and returned it as precise_address — on this layout that is map +// furniture, so `findmy person X --zoom` reported place names and the map's +// own "3D" control as the person's street address. Wrong location data +// presented as precise is worse than no data. +var ErrNoDetailPane = errors.New("no FindMy detail pane found right of the sidebar") + // ExtractDetailPaneAddress filters OCR lines to FindMy's right-side detail // pane and extracts the address rendered below the selected person/device // header. US addresses are split into city/region/postal when possible; // otherwise the visible address lines are returned as a single precise address. -func ExtractDetailPaneAddress(lines []TextLine, sidebarRightPx int) (precise, city, region, postal string) { +// +// entityName is the name of the row that was clicked. The detail pane always +// headers with it, and a map never renders it, so requiring the header to +// match is the language-neutral proof that a detail pane is really on screen. +// A returned ErrNoDetailPane means the layout has no detail pane at all; +// a nil error with an empty precise means the pane is there but carries no +// address (offline device, "No location found"). +func ExtractDetailPaneAddress(lines []TextLine, sidebarRightPx int, entityName string) (precise, city, region, postal string, err error) { rightPane := make([]TextLine, 0, len(lines)) for _, l := range lines { txt := strings.TrimSpace(l.Text) @@ -693,6 +739,7 @@ func ExtractDetailPaneAddress(lines []TextLine, sidebarRightPx int) (precise, ci addressLines := make([]string, 0, 3) seenHeader := false + headerX := 0 for _, l := range rightPane { txt := normalizeDetailPaneText(l.Text) if txt == "" { @@ -705,10 +752,27 @@ func ExtractDetailPaneAddress(lines []TextLine, sidebarRightPx int) (precise, ci continue } if !seenHeader { + // Only the entity's own name opens the pane. Anything else here + // is map canvas, and consuming it as a header is what let map + // labels through as addresses. + if !matchesEntityHeader(txt, entityName) { + continue + } seenHeader = true + headerX = l.X continue } - if len(addressLines) > 0 && !looksLikeAddressLine(txt) { + // Address lines are left-aligned under the header in a real pane. + // Map labels are scattered across the canvas, so an X far from the + // header's is map furniture no matter how much it reads like a + // street ("WAREHAM LN", "BULL RUN RD"). + if abs(l.X-headerX) > detailPaneColumnTolerancePx { + break + } + // The first address line is held to the same shape test as the rest. + // Without this a pane whose address had not painted yet would adopt + // the next unrelated label. + if !looksLikeAddressLine(txt) { break } addressLines = append(addressLines, txt) @@ -720,18 +784,53 @@ func ExtractDetailPaneAddress(lines []TextLine, sidebarRightPx int) (precise, ci } } + if !seenHeader { + return "", "", "", "", ErrNoDetailPane + } if len(addressLines) == 0 { - return "", "", "", "" + return "", "", "", "", nil } for i, line := range addressLines { if c, r, p, ok := parseCityRegionPostal(line); ok { if i == 0 { - return line, c, r, p + return line, c, r, p, nil } - return strings.Join(addressLines[:i], ", "), c, r, p + return strings.Join(addressLines[:i], ", "), c, r, p, nil + } + } + return strings.Join(addressLines, ", "), "", "", "", nil +} + +// matchesEntityHeader reports whether an OCR line is the detail pane's header +// for entityName. It is deliberately tolerant: Vision runs with language +// correction off (it mangles proper nouns otherwise) but still drops or +// garbles the occasional character, and the header may be truncated with an +// ellipsis when the pane is narrow. Half the name's words matching is enough +// to tell a header apart from a map label, which shares no words at all. +func matchesEntityHeader(line, entityName string) bool { + name := strings.ToLower(normalizeDetailPaneText(entityName)) + candidate := strings.ToLower(normalizeDetailPaneText(line)) + if name == "" || candidate == "" { + return false + } + if candidate == name || strings.Contains(candidate, name) || strings.Contains(name, candidate) { + return true + } + nameWords := strings.Fields(name) + if len(nameWords) == 0 { + return false + } + candidateWords := make(map[string]bool, 8) + for _, w := range strings.Fields(candidate) { + candidateWords[w] = true + } + hits := 0 + for _, w := range nameWords { + if candidateWords[w] { + hits++ } } - return strings.Join(addressLines, ", "), "", "", "" + return hits*2 >= len(nameWords) } func normalizeDetailPaneText(s string) string { @@ -746,7 +845,21 @@ func parseCityRegionPostal(s string) (city, region, postal string, ok bool) { return m[1], m[2], m[3], true } +var streetWords = map[string]bool{ + "street": true, "st": true, "avenue": true, "ave": true, + "road": true, "rd": true, "drive": true, "dr": true, + "lane": true, "ln": true, "way": true, "place": true, + "pl": true, "court": true, "ct": true, "boulevard": true, "blvd": true, +} + func looksLikeAddressLine(s string) bool { + // FindMy joins a coarse location to its staleness with a bullet + // ("Winston-Salem, NC • Now"). A street address never contains one, and + // this is exactly the text the redesigned map callout renders under the + // name — the coarse reading we already have, not a precise address. + if strings.ContainsAny(s, "•·") { + return false + } if _, _, _, ok := parseCityRegionPostal(s); ok { return true } @@ -755,9 +868,11 @@ func looksLikeAddressLine(s string) bool { return true } } - t := strings.ToLower(s) - for _, word := range []string{"street", "st", "avenue", "ave", "road", "rd", "drive", "dr", "lane", "ln", "way", "place", "pl", "court", "ct", "boulevard", "blvd"} { - if strings.Contains(t, word) { + // Whole words only. Substring matching treated any word containing "st", + // "dr", "ln" and friends as a street — "Winston-Salem" and "Redmond" both + // qualified — which is how map labels passed for addresses. + for _, word := range strings.Fields(strings.ToLower(s)) { + if streetWords[strings.Trim(word, ".,'\"()")] { return true } }