From 39cff664baaf5a7c960cd4ba96ac15ce4401844c Mon Sep 17 00:00:00 2001 From: William Harris Date: Thu, 5 Mar 2026 06:07:23 +0000 Subject: [PATCH 1/5] docs: plan --- docs/dev_todo/data_sync_improvements.md | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/dev_todo/data_sync_improvements.md diff --git a/docs/dev_todo/data_sync_improvements.md b/docs/dev_todo/data_sync_improvements.md new file mode 100644 index 00000000..04c5b356 --- /dev/null +++ b/docs/dev_todo/data_sync_improvements.md @@ -0,0 +1,89 @@ +# Data Sync Improvements + +**GitHub Issue:** [#260](https://github.com/williscool/CalendarNotification/issues/260) + +## Overview + +Formalize the "delete all + re-upload" workflow as the primary sync action and add a progress indicator so you know when sync is truly complete. This is critical because a downstream app consumes the synced events (and their snooze targets) to reschedule them in bulk — incorrect or incomplete data means events get rescheduled wrong. + +## Background + +### How sync works today + +1. User opens "Data Sync" from the Android menu → launches React Native `SetupSync` screen +2. Taps "Sync Events Local To PowerSync Now" → `psInsertDbTable` copies all rows from the local SQLite `eventsV9` into the PowerSync local DB via `INSERT OR REPLACE` +3. PowerSync SDK detects the pending CRUD changes and uploads them to Supabase Postgres via `Connector.uploadData()` (with retries and error handling) +4. "Clear Remote PowerSync Events" is a separate button hidden in the Danger Zone — runs `DELETE FROM eventsV9` on the PowerSync DB, which then propagates deletes to Supabase + +### The problems + +**No sync completion indicator.** PowerSync does a lot of background work before data is fully reflected in Supabase Postgres. The only way to know sync is done is to watch the raw `currentStatus` JSON in the debug UI and wait for operation counts to stop changing. There is no dedicated progress display. + +**Incremental updates don't work reliably.** When events get updated (particularly around dismissed events lingering), the `INSERT OR REPLACE` approach produces unexpected results. The practical workaround is to clear all remote events first, then re-upload — but this requires manually opening the Danger Zone, clearing, then syncing. This is what you end up doing every time. + +### History of related issues + +| Issue | Status | Summary | +|-------|--------|---------| +| [#47](https://github.com/williscool/CalendarNotification/issues/47) | Closed | Original feature request — sync local DB to remote Postgres for analytics | +| [#86](https://github.com/williscool/CalendarNotification/issues/86) | Closed | Delete should use remote truncate; turned out the real problem was no error handling in `uploadData`; closed in favor of #93 | +| [#93](https://github.com/williscool/CalendarNotification/issues/93) | Open | Need to verify sync completion before trusting the data; credential UX (done) | +| [#245](https://github.com/williscool/CalendarNotification/issues/245) | Closed | Event count mismatch (138 local vs 135 remote); fixed in #247 | +| [#246](https://github.com/williscool/CalendarNotification/issues/246) | Closed | Snoozed events not syncing due to `displayStatus` filter bug | + +The `uploadData` error handling and retry logic was added to `Connector.ts` after #86. The `displayStatus` filter bug (#246) was fixed by removing the `WHERE dsts != 0` filter (see [fix_sync_display_status_filter.md](./fix_sync_display_status_filter.md)). Despite these fixes, the fundamental issue remains: PowerSync takes real time to process and there's no good feedback loop. + +### Key technical detail for progress tracking + +PowerSync exposes the pending upload queue size via: + +```sql +SELECT COUNT(*) FROM ps_crud; +``` + +Combined with `currentStatus.dataFlowStatus.uploading` (boolean), this gives us everything needed for a progress indicator. + +## Plan + +### Phase 1: Formalize "Delete + Re-upload" as the primary sync action + +Combine the current two-step manual workflow (Danger Zone clear → sync) into the primary sync button. The button becomes a full resync: clear remote, then re-upload all active events. + +**Files involved:** +- `src/lib/orm/index.ts` — new combined resync function +- `src/lib/features/SetupSync.tsx` — update `handleSync` to call the combined function, update button label + +The individual "Clear Remote PowerSync Events" button stays in the Danger Zone for manual use. + +### Phase 2: Sync progress indicator + +Replace the raw `currentStatus` JSON dump with a human-readable progress display: + +1. When sync starts, button goes disabled/loading +2. Poll `ps_crud` count to show "X operations remaining" +3. Use `dataFlowStatus.uploading` to distinguish "queued" vs "actively uploading" +4. When count hits 0 and uploading is false → show "Sync complete!" with timestamp +5. Re-enable button + +**Files involved:** +- `src/lib/features/SetupSync.tsx` — progress state, polling, status banner + +### Phase 3 (future): Investigate update weirdness + +The likely culprit is dismissed events that linger in `eventsV9` and don't get cleaned up, causing stale data in the remote DB after incremental sync. The delete+reupload approach from Phase 1 sidesteps this entirely, but if we ever want true incremental sync it will need investigation. + +## Related Files + +| File | Role | +|------|------| +| `src/lib/features/SetupSync.tsx` | Main sync UI — sync button, danger zone, status display | +| `src/lib/orm/index.ts` | `psInsertDbTable`, `psClearTable` — the sync and clear primitives | +| `src/lib/powersync/Connector.ts` | `uploadData` with retry logic, `emitSyncLog` | +| `src/screens/sync-debug.tsx` | Debug screen with raw status, logs, failed ops | +| `src/lib/powersync/index.tsx` | PowerSync DB setup, `setupPowerSync()` | + +## References + +- [fix_sync_display_status_filter.md](./fix_sync_display_status_filter.md) — the `displayStatus` filter bug and fix +- [sync_database_mismatch.md](../dev_completed/sync_database_mismatch.md) — Room vs Legacy DB name mismatch fix +- [PowerSync SyncStatus docs](https://powersync-ja.github.io/powersync-js/react-native-sdk/classes/SyncStatus) — `dataFlowStatus`, `uploading`, `hasSynced` From 932f01d6bec4365dca6abc2c8ab0fd6d04a31437 Mon Sep 17 00:00:00 2001 From: William Harris Date: Fri, 6 Mar 2026 02:33:31 +0000 Subject: [PATCH 2/5] feat: init commit of progress --- src/lib/features/SetupSync.tsx | 38 ++++++++++++++++++++++++++++------ src/lib/orm/index.ts | 24 +++++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/lib/features/SetupSync.tsx b/src/lib/features/SetupSync.tsx index 98161f52..3656bcef 100644 --- a/src/lib/features/SetupSync.tsx +++ b/src/lib/features/SetupSync.tsx @@ -5,7 +5,7 @@ import { open } from '@op-engineering/op-sqlite'; import { useQuery } from '@powersync/react'; import { PowerSyncContext } from "@powersync/react"; import { installCrsqliteOnTable } from '@lib/cr-sqlite/install'; -import { psInsertDbTable, psClearTable } from '@lib/orm'; +import { psResyncTable, psClearTable, getPendingCrudCount } from '@lib/orm'; import { useNavigation } from '@react-navigation/native'; import type { AppNavigationProp } from '@lib/navigation/types'; import { useSettings } from '@lib/hooks/SettingsContext'; @@ -52,6 +52,8 @@ export const SetupSync = () => { const [showDangerZone, setShowDangerZone] = useState(false); const [showDebugOutput, setShowDebugOutput] = useState(false); const [isConnected, setIsConnected] = useState(null); + const [pendingOps, setPendingOps] = useState(null); + const isSyncing = pendingOps !== null && pendingOps > 0; const isConfigured = isSettingsConfigured(settings); @@ -105,12 +107,12 @@ export const SetupSync = () => { // Track previous values to avoid unnecessary re-renders for expensive updates let prevStatus = ''; let prevConnected: boolean | null = null; + let prevPendingOps: number | null = null; - const statusInterval = setInterval(() => { + const statusInterval = setInterval(async () => { if (providerDb) { const newStatus = JSON.stringify(providerDb.currentStatus); - // Only update dbStatus if it actually changed (avoid expensive re-render) if (newStatus !== prevStatus) { prevStatus = newStatus; setDbStatus(newStatus); @@ -123,6 +125,20 @@ export const SetupSync = () => { setIsConnected(newConnected); } } + + // Always poll pending ops — survives navigation away and back + try { + const count = await getPendingCrudCount(providerDb); + if (count !== prevPendingOps) { + prevPendingOps = count; + setPendingOps(count); + if (count === 0 && prevPendingOps !== null) { + emitSyncLog('info', 'Sync complete — upload queue drained'); + } + } + } catch (error) { + emitSyncLog('warn', 'Failed to poll pending ops count', { error }); + } } }, 1000); @@ -133,7 +149,7 @@ export const SetupSync = () => { if (!providerDb || !settings.syncEnabled) return; try { - await psInsertDbTable(eventsDbName, 'eventsV9', providerDb); + await psResyncTable(eventsDbName, 'eventsV9', providerDb); const result = await regDb.execute(debugDisplayQuery); if (result?.rows) { setSqliteEvents(result.rows || []); @@ -233,13 +249,23 @@ export const SetupSync = () => { )} + {isSyncing && ( + + + {pendingOps !== null + ? `Uploading — ${pendingOps} operation${pendingOps !== 1 ? 's' : ''} remaining` + : 'Preparing sync...'} + + + )} + - Sync Events Local To PowerSync Now + {isSyncing ? 'Syncing...' : 'Full Resync to Remote'} { + emitSyncLog('info', `Starting full resync for ${tableName}`); + await psClearTable(tableName, psDb); + await psInsertDbTable(dbName, tableName, psDb); + emitSyncLog('info', `Full resync queued for ${tableName}`); +} + +/** Returns the number of pending CRUD operations in the PowerSync upload queue */ +export async function getPendingCrudCount( + psDb: AbstractPowerSyncDatabase +): Promise { + const result = await psDb.execute('SELECT COUNT(*) AS cnt FROM ps_crud'); + const row = result?.rows?.item(0); + return (row?.cnt as number) ?? 0; +} + From fcc51daa24f5b31d88b0e9784a8dd99eef8a3ae0 Mon Sep 17 00:00:00 2001 From: William Harris Date: Fri, 6 Mar 2026 02:49:19 +0000 Subject: [PATCH 3/5] fix: much better progress --- src/lib/features/SetupSync.tsx | 19 ++++++++++++------- src/lib/powersync/Connector.ts | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/lib/features/SetupSync.tsx b/src/lib/features/SetupSync.tsx index 3656bcef..c9e8150b 100644 --- a/src/lib/features/SetupSync.tsx +++ b/src/lib/features/SetupSync.tsx @@ -6,6 +6,8 @@ import { useQuery } from '@powersync/react'; import { PowerSyncContext } from "@powersync/react"; import { installCrsqliteOnTable } from '@lib/cr-sqlite/install'; import { psResyncTable, psClearTable, getPendingCrudCount } from '@lib/orm'; +import { getUploadProgress, resetUploadProgress } from '@lib/powersync/Connector'; +import type { UploadProgress } from '@lib/powersync/Connector'; import { useNavigation } from '@react-navigation/native'; import type { AppNavigationProp } from '@lib/navigation/types'; import { useSettings } from '@lib/hooks/SettingsContext'; @@ -53,6 +55,7 @@ export const SetupSync = () => { const [showDebugOutput, setShowDebugOutput] = useState(false); const [isConnected, setIsConnected] = useState(null); const [pendingOps, setPendingOps] = useState(null); + const [uploadProgress, setUploadProgress] = useState({ upserts: 0, updates: 0, deletes: 0 }); const isSyncing = pendingOps !== null && pendingOps > 0; const isConfigured = isSettingsConfigured(settings); @@ -126,16 +129,18 @@ export const SetupSync = () => { } } - // Always poll pending ops — survives navigation away and back + // Always poll pending ops + uploaded counter — survives navigation try { const count = await getPendingCrudCount(providerDb); if (count !== prevPendingOps) { - prevPendingOps = count; - setPendingOps(count); - if (count === 0 && prevPendingOps !== null) { + if (count === 0 && prevPendingOps !== null && prevPendingOps > 0) { emitSyncLog('info', 'Sync complete — upload queue drained'); + resetUploadProgress(); } + prevPendingOps = count; + setPendingOps(count); } + setUploadProgress(getUploadProgress()); } catch (error) { emitSyncLog('warn', 'Failed to poll pending ops count', { error }); } @@ -149,6 +154,8 @@ export const SetupSync = () => { if (!providerDb || !settings.syncEnabled) return; try { + resetUploadProgress(); + setUploadProgress({ upserts: 0, updates: 0, deletes: 0 }); await psResyncTable(eventsDbName, 'eventsV9', providerDb); const result = await regDb.execute(debugDisplayQuery); if (result?.rows) { @@ -252,9 +259,7 @@ export const SetupSync = () => { {isSyncing && ( - {pendingOps !== null - ? `Uploading — ${pendingOps} operation${pendingOps !== 1 ? 's' : ''} remaining` - : 'Preparing sync...'} + {`${uploadProgress.deletes} deleted, ${uploadProgress.upserts} upserted, ${uploadProgress.updates} updated — ${pendingOps} queued`} )} diff --git a/src/lib/powersync/Connector.ts b/src/lib/powersync/Connector.ts index 432d9982..144002fa 100644 --- a/src/lib/powersync/Connector.ts +++ b/src/lib/powersync/Connector.ts @@ -110,6 +110,20 @@ const BASE_DELAY_MS = 1000; const MAX_FAILED_OPS = 50; const FAILED_OPS_STORAGE_KEY = '@powersync_failed_operations'; +// Live upload progress counters — incremented per successful op in uploadData +export interface UploadProgress { + upserts: number; + updates: number; + deletes: number; +} +const _uploadProgress: UploadProgress = { upserts: 0, updates: 0, deletes: 0 }; +export const getUploadProgress = (): UploadProgress => ({ ..._uploadProgress }); +export const resetUploadProgress = (): void => { + _uploadProgress.upserts = 0; + _uploadProgress.updates = 0; + _uploadProgress.deletes = 0; +}; + interface SupabaseError { code: string; message?: string; @@ -368,6 +382,9 @@ export class Connector implements PowerSyncBackendConnector { for (const op of transaction.crud) { lastOp = op; await this.executeWithRetry(op); + if (op.op === UpdateType.PUT) _uploadProgress.upserts++; + else if (op.op === UpdateType.PATCH) _uploadProgress.updates++; + else if (op.op === UpdateType.DELETE) _uploadProgress.deletes++; } await transaction.complete(); From 5dc05cf63c8c9d354abe00113e8e5bb312ff412b Mon Sep 17 00:00:00 2001 From: William Harris Date: Fri, 6 Mar 2026 03:28:48 +0000 Subject: [PATCH 4/5] fix: little code review stuff --- src/lib/features/SetupSync.tsx | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/lib/features/SetupSync.tsx b/src/lib/features/SetupSync.tsx index c9e8150b..587d6440 100644 --- a/src/lib/features/SetupSync.tsx +++ b/src/lib/features/SetupSync.tsx @@ -56,6 +56,7 @@ export const SetupSync = () => { const [isConnected, setIsConnected] = useState(null); const [pendingOps, setPendingOps] = useState(null); const [uploadProgress, setUploadProgress] = useState({ upserts: 0, updates: 0, deletes: 0 }); + const [syncCompleteAt, setSyncCompleteAt] = useState(null); const isSyncing = pendingOps !== null && pendingOps > 0; const isConfigured = isSettingsConfigured(settings); @@ -111,6 +112,7 @@ export const SetupSync = () => { let prevStatus = ''; let prevConnected: boolean | null = null; let prevPendingOps: number | null = null; + let prevUploadProgress = ''; const statusInterval = setInterval(async () => { if (providerDb) { @@ -135,12 +137,17 @@ export const SetupSync = () => { if (count !== prevPendingOps) { if (count === 0 && prevPendingOps !== null && prevPendingOps > 0) { emitSyncLog('info', 'Sync complete — upload queue drained'); - resetUploadProgress(); + setSyncCompleteAt(new Date().toLocaleTimeString()); } prevPendingOps = count; setPendingOps(count); } - setUploadProgress(getUploadProgress()); + const progress = getUploadProgress(); + const progressKey = `${progress.upserts},${progress.updates},${progress.deletes}`; + if (progressKey !== prevUploadProgress) { + prevUploadProgress = progressKey; + setUploadProgress(progress); + } } catch (error) { emitSyncLog('warn', 'Failed to poll pending ops count', { error }); } @@ -156,6 +163,7 @@ export const SetupSync = () => { try { resetUploadProgress(); setUploadProgress({ upserts: 0, updates: 0, deletes: 0 }); + setSyncCompleteAt(null); await psResyncTable(eventsDbName, 'eventsV9', providerDb); const result = await regDb.execute(debugDisplayQuery); if (result?.rows) { @@ -264,6 +272,14 @@ export const SetupSync = () => { )} + {!isSyncing && syncCompleteAt && ( + + + {`Sync complete at ${syncCompleteAt}`} + + + )} + Date: Fri, 6 Mar 2026 03:40:56 +0000 Subject: [PATCH 5/5] fix: tap gap Sync button not disabled during async resync operation Medium Severity The isSyncing flag is derived solely from pendingOps > 0, which is updated by a 1-second polling interval. When the user clicks "Full Resync," handleSync runs asynchronously but isSyncing remains false until the next poll detects queued CRUD operations. During that gap (up to ~1 second plus the psResyncTable execution time), the button stays enabled. A second click triggers a concurrent psResyncTable, whose psClearTable can run between the first call's psClearTable and psInsertDbTable, deleting freshly inserted rows and producing an incomplete sync. --- src/lib/features/SetupSync.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/features/SetupSync.tsx b/src/lib/features/SetupSync.tsx index 587d6440..d996dbdf 100644 --- a/src/lib/features/SetupSync.tsx +++ b/src/lib/features/SetupSync.tsx @@ -57,7 +57,8 @@ export const SetupSync = () => { const [pendingOps, setPendingOps] = useState(null); const [uploadProgress, setUploadProgress] = useState({ upserts: 0, updates: 0, deletes: 0 }); const [syncCompleteAt, setSyncCompleteAt] = useState(null); - const isSyncing = pendingOps !== null && pendingOps > 0; + const [syncInFlight, setSyncInFlight] = useState(false); + const isSyncing = syncInFlight || (pendingOps !== null && pendingOps > 0); const isConfigured = isSettingsConfigured(settings); @@ -160,6 +161,7 @@ export const SetupSync = () => { const handleSync = async () => { if (!providerDb || !settings.syncEnabled) return; + setSyncInFlight(true); try { resetUploadProgress(); setUploadProgress({ upserts: 0, updates: 0, deletes: 0 }); @@ -171,6 +173,8 @@ export const SetupSync = () => { } } catch (error) { emitSyncLog('error', 'Failed to sync data', { error }); + } finally { + setSyncInFlight(false); } };