From a3207e7daa14734b2c3cb2d5f7f1238e5c52c069 Mon Sep 17 00:00:00 2001 From: DTTerastar Date: Mon, 20 Apr 2026 21:58:10 -0400 Subject: [PATCH 1/2] Add --derive flag to polyfill sleep bounds from intraday samples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For accounts whose only recent overnight source is an Apple Watch via HealthKit, Withings stores the raw intraday signal but never computes a sleep summary. `sleep --derive` fills the gap: for each date in the --since window with no getsummary record, query getintradayactivity for prior-day 18:00 → current-day 12:00 local, find the longest contiguous quiet run (heart_rate ≤ 80, steps == 0, gap tolerance 60 min), and emit a derived row if the run is ≥ 3h. Adds a new `source` column to CSV/JSON: "summary" for API rows, "derived" for polyfilled rows. Derived rows populate start/end, total_sleep_min, and hr_avg; stage breakdown and sleep_score stay empty since we can't infer them. Throttles intraday calls at 250ms between days to stay under Withings' per-minute rate cap when deriving long windows. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/sleep.go | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 2 deletions(-) diff --git a/cmd/sleep.go b/cmd/sleep.go index 0a6901d..777af93 100644 --- a/cmd/sleep.go +++ b/cmd/sleep.go @@ -21,6 +21,9 @@ type sleepSeries struct { EndDate int64 `json:"enddate"` Date string `json:"date"` Data json.RawMessage `json:"data"` + // Source is synthetic (not from API): "summary" for getsummary rows, + // "derived" for rows polyfilled from intraday samples via --derive. + Source string `json:"source"` } type sleepResponse struct { @@ -30,8 +33,9 @@ type sleepResponse struct { } var ( - sleepJSONFlag bool - sleepSinceFlag string + sleepJSONFlag bool + sleepSinceFlag string + sleepDeriveFlag bool ) var sleepCmd = &cobra.Command{ @@ -69,6 +73,36 @@ var sleepCmd = &cobra.Command{ params.Set("offset", strconv.Itoa(resp.Offset)) } + haveDate := make(map[string]bool, len(all)) + for i := range all { + all[i].Source = "summary" + haveDate[all[i].Date] = true + } + + if sleepDeriveFlag { + today := time.Now() + first := true + for d := since; !d.After(today); d = d.AddDate(0, 0, 1) { + dateStr := d.Format("2006-01-02") + if haveDate[dateStr] { + continue + } + // Throttle — Withings rate-limits aggressive callers (status 601). + // 250ms between calls keeps a wide window-derive under the cap. + if !first { + time.Sleep(250 * time.Millisecond) + } + first = false + derived, err := deriveSleep(c, d) + if err != nil { + return err + } + if derived != nil { + all = append(all, *derived) + } + } + } + sort.Slice(all, func(i, j int) bool { return all[i].StartDate < all[j].StartDate }) if sleepJSONFlag { @@ -78,6 +112,129 @@ var sleepCmd = &cobra.Command{ }, } +// deriveSleep polyfills a sleep start/end for a night that has no getsummary +// record, by finding the longest contiguous "quiet" run of intraday samples. +// +// Window: prior-day 18:00 local → current-day 12:00 local (18h — fits in one +// 24h getintradayactivity call). A sample is "quiet" when heart_rate is +// present, ≤ 80 bpm, and steps == 0. Runs tolerate gaps ≤ 60 min between +// consecutive quiet samples (watch off for charging). The longest qualifying +// run ≥ 3h becomes the derived sleep session. Returns (nil, nil) when no +// window qualifies or intraday has no samples. +func deriveSleep(c *client.Client, date time.Time) (*sleepSeries, error) { + loc := time.Local + day := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc) + winStart := day.AddDate(0, 0, -1).Add(18 * time.Hour) + winEnd := day.Add(12 * time.Hour) + + params := url.Values{} + params.Set("action", "getintradayactivity") + params.Set("startdate", strconv.FormatInt(winStart.Unix(), 10)) + params.Set("enddate", strconv.FormatInt(winEnd.Unix(), 10)) + params.Set("data_fields", "steps,heart_rate,duration") + + var resp intradayResponse + if err := c.Call("/v2/measure", params, &resp); err != nil { + return nil, err + } + + type sample struct { + ts int64 + hr int + steps int + } + samples := make([]sample, 0, len(resp.Series)) + for tsStr, p := range resp.Series { + ts, err := strconv.ParseInt(tsStr, 10, 64) + if err != nil { + continue + } + samples = append(samples, sample{ts: ts, hr: p.HeartRate, steps: p.Steps}) + } + sort.Slice(samples, func(i, j int) bool { return samples[i].ts < samples[j].ts }) + if len(samples) == 0 { + return nil, nil + } + + const maxGap int64 = 60 * 60 // 60 min — tolerate watch-off-for-charging + const minDur int64 = 3 * 3600 // 3h — ignore naps, noise + + isQuiet := func(s sample) bool { return s.hr > 0 && s.hr <= 80 && s.steps == 0 } + + var bestStart, bestEnd, bestDur int64 + var bestHRSum, bestHRCount int + + var runStart, runEnd, lastTS int64 + var runHRSum, runHRCount int + inRun := false + + closeRun := func() { + dur := runEnd - runStart + if dur > bestDur { + bestDur = dur + bestStart = runStart + bestEnd = runEnd + bestHRSum = runHRSum + bestHRCount = runHRCount + } + } + + for _, s := range samples { + switch { + case isQuiet(s) && !inRun: + runStart, runEnd = s.ts, s.ts + runHRSum, runHRCount = s.hr, 1 + inRun = true + case isQuiet(s) && s.ts-lastTS > maxGap: + closeRun() + runStart, runEnd = s.ts, s.ts + runHRSum, runHRCount = s.hr, 1 + case isQuiet(s): + runEnd = s.ts + runHRSum += s.hr + runHRCount++ + default: + if inRun { + closeRun() + inRun = false + } + } + if isQuiet(s) { + lastTS = s.ts + } + } + if inRun { + closeRun() + } + + if bestDur < minDur { + return nil, nil + } + + hrAvg := 0.0 + if bestHRCount > 0 { + hrAvg = float64(bestHRSum) / float64(bestHRCount) + } + + // Stuff total duration into lightsleepduration so the existing CSV writer's + // total_sleep_min = (light+deep+rem)/60 renders correctly. Derived rows + // have no stage breakdown, so all sleep time shows as "light". + dataObj := map[string]any{ + "lightsleepduration": bestDur, + "hr_average": hrAvg, + } + data, _ := json.Marshal(dataObj) + + return &sleepSeries{ + Timezone: loc.String(), + StartDate: bestStart, + EndDate: bestEnd, + Date: date.Format("2006-01-02"), + Data: json.RawMessage(data), + Source: "derived", + }, nil +} + func writeSleepCSV(series []sleepSeries) error { w := csv.NewWriter(os.Stdout) defer w.Flush() @@ -87,6 +244,7 @@ func writeSleepCSV(series []sleepSeries) error { "time_to_sleep_sec", "time_to_wakeup_sec", "wakeup_count", "sleep_score", "hr_avg", "hr_min", "hr_max", "rr_avg", "rr_min", "rr_max", "snore_episodes", "apnea_hypopnea_index", + "source", } if err := w.Write(header); err != nil { return err @@ -138,6 +296,7 @@ func writeSleepCSV(series []sleepSeries) error { strconv.FormatFloat(d.RRMax, 'f', -1, 64), strconv.Itoa(d.SnoreEpisodes), strconv.FormatFloat(d.ApneaHypopnea, 'f', -1, 64), + s.Source, } if err := w.Write(row); err != nil { return err @@ -151,4 +310,6 @@ func init() { "Filter on or after date (e.g. 2026-01-01, 30d, 4w, 6m, 1y; default 30d)") sleepCmd.Flags().BoolVar(&sleepJSONFlag, "json", false, "Output as JSON instead of CSV") + sleepCmd.Flags().BoolVar(&sleepDeriveFlag, "derive", false, + "For nights with no Withings sleep summary, polyfill start/end from intraday heart-rate samples") } From b374b595a3f82c5dc3dfdfb16a261702221e4464 Mon Sep 17 00:00:00 2001 From: DTTerastar Date: Mon, 20 Apr 2026 22:07:45 -0400 Subject: [PATCH 2/2] Loosen sleep-derive heuristic to stop fragmenting on step-only samples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apple Watch via HealthKit reports small step-only samples during the night (bathroom visits, rolling over, 15-30 steps each) with heart_rate=0. The previous `heart_rate > 0 AND steps == 0` requirement disqualified these, cutting real 8h sleep into 3-4h fragments. Real case: user's 8h01 actual was being detected as 3h32. Switch to a break/continue model: a sample only BREAKS a run if it indicates real wakefulness (hr > 90 or steps > 30). Step-only samples with modest counts pass through. Accept the longest run that is ≥ 3h, has ≥ 10 HR readings, and whose HR readings average ≤ 80 bpm — enough to confirm the watch was on and the person was resting. Also bumps the watch-off-for-charging gap tolerance from 60 to 90 min, reflecting that a 1-hour charge during sleep is common. On the same account: Apr 18 now 6h25 (actual 6h59) and Apr 19 now 9h27 (actual 8h01) — vs the prior 4h47 / 3h32. Residual overshoot on Apr 19 is time-in-bed-awake that the heuristic can't separate from sleep without movement data. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/sleep.go | 115 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 42 deletions(-) diff --git a/cmd/sleep.go b/cmd/sleep.go index 777af93..bd737f9 100644 --- a/cmd/sleep.go +++ b/cmd/sleep.go @@ -156,65 +156,96 @@ func deriveSleep(c *client.Client, date time.Time) (*sleepSeries, error) { return nil, nil } - const maxGap int64 = 60 * 60 // 60 min — tolerate watch-off-for-charging - const minDur int64 = 3 * 3600 // 3h — ignore naps, noise - - isQuiet := func(s sample) bool { return s.hr > 0 && s.hr <= 80 && s.steps == 0 } - - var bestStart, bestEnd, bestDur int64 - var bestHRSum, bestHRCount int - - var runStart, runEnd, lastTS int64 - var runHRSum, runHRCount int + // A sample "breaks" a sleep run when it shows real wakefulness: elevated + // heart rate, or a meaningful burst of steps. Small step samples (bathroom + // trips, rolling over) do not break the run — they're just HealthKit's way + // of reporting brief motion without an accompanying HR reading. + const hrBreak = 90 // bpm above this = awake/active + const stepsBreak = 30 + + const maxGap int64 = 90 * 60 // 90 min — tolerate watch-off-for-charging + const minDur int64 = 3 * 3600 // 3h — ignore naps + + isBreak := func(s sample) bool { return s.hr > hrBreak || s.steps > stepsBreak } + + type run struct { + start, end int64 + hrSum int + hrCount int + } + var runs []run + var cur run inRun := false + var lastTS int64 - closeRun := func() { - dur := runEnd - runStart - if dur > bestDur { - bestDur = dur - bestStart = runStart - bestEnd = runEnd - bestHRSum = runHRSum - bestHRCount = runHRCount + startRun := func(s sample) { + cur = run{start: s.ts, end: s.ts} + if s.hr > 0 { + cur.hrSum = s.hr + cur.hrCount = 1 + } + inRun = true + } + extendRun := func(s sample) { + cur.end = s.ts + if s.hr > 0 { + cur.hrSum += s.hr + cur.hrCount++ } } + flushRun := func() { + runs = append(runs, cur) + inRun = false + } for _, s := range samples { - switch { - case isQuiet(s) && !inRun: - runStart, runEnd = s.ts, s.ts - runHRSum, runHRCount = s.hr, 1 - inRun = true - case isQuiet(s) && s.ts-lastTS > maxGap: - closeRun() - runStart, runEnd = s.ts, s.ts - runHRSum, runHRCount = s.hr, 1 - case isQuiet(s): - runEnd = s.ts - runHRSum += s.hr - runHRCount++ - default: + if isBreak(s) { if inRun { - closeRun() - inRun = false + flushRun() } + continue } - if isQuiet(s) { - lastTS = s.ts + switch { + case !inRun: + startRun(s) + case s.ts-lastTS > maxGap: + flushRun() + startRun(s) + default: + extendRun(s) } + lastTS = s.ts } if inRun { - closeRun() + flushRun() } - if bestDur < minDur { + // Accept the longest run that looks like sleep: ≥ 3h, with enough HR data + // to confirm it's not just "watch not worn," and a mean HR in the sleep range. + var best *run + for i := range runs { + r := &runs[i] + if r.end-r.start < minDur { + continue + } + if r.hrCount < 10 { + continue + } + if float64(r.hrSum)/float64(r.hrCount) > 80 { + continue + } + if best == nil || r.end-r.start > best.end-best.start { + best = r + } + } + if best == nil { return nil, nil } - hrAvg := 0.0 - if bestHRCount > 0 { - hrAvg = float64(bestHRSum) / float64(bestHRCount) - } + bestStart := best.start + bestEnd := best.end + bestDur := best.end - best.start + hrAvg := float64(best.hrSum) / float64(best.hrCount) // Stuff total duration into lightsleepduration so the existing CSV writer's // total_sleep_min = (light+deep+rem)/60 renders correctly. Derived rows