From 084091653afba8903b9274cb5a7123cf846ba5bc Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 14 Nov 2026 11:22:34 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(web):=20paginate=20discussion=20comment?= =?UTF-8?q?=20imports=20=E2=80=94=20long=20threads=20lost=20every=20new=20?= =?UTF-8?q?comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident 2026-11-14 (owner-reported): awesome-dsh-plugin/awesome-dsh-plugin discussion 5795 stopped waking any worker after its comment count crossed 50. importDiscussionComments fetched only the FIRST page (/discussions/N/comments?per_page=50, ascending = oldest 50); newer comments sat on page 2+, were never imported, and the updated_at cursor still advanced past them so they could never be retried. The issue-comment variant got the same walk-every-page fix on 2026-11-07 — the discussion variant was missed. Fix: same loop as importIssueComments (per_page=100, walk until a short page, 50-page safety cap). 3 comments were lost (11-10/11-13); they are recovered by resetting the discussion cursor after deploy — re-import is idempotent via getCommentByUpstreamId and re-emits comment webhooks so workers wake normally. Tests: web suite 275 pass / 0 fail; tsc clean. --- packages/web/package.json | 2 +- packages/web/src/upstream-sync.ts | 38 +++++++++++++++++++------------ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/packages/web/package.json b/packages/web/package.json index d47480a..54115eb 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,6 +1,6 @@ { "name": "ework-web", - "version": "0.10.200", + "version": "0.10.201", "type": "module", "description": "ework-web — standalone multi-project issue tracker. Local SQLite-backed, no external API dependency. Bun + TypeScript + SSR HTML.", "license": "MIT", diff --git a/packages/web/src/upstream-sync.ts b/packages/web/src/upstream-sync.ts index 7f42dbf..7fb5c39 100644 --- a/packages/web/src/upstream-sync.ts +++ b/packages/web/src/upstream-sync.ts @@ -1411,22 +1411,30 @@ export class UpstreamSync { private async importDiscussionComments(d: GithubDiscussion, emit: boolean): Promise { const local = await getIssueByUpstreamNumber(this.project.id, d.number, "discussion"); if (!local) return 0; - const comments = await this.fetchJson(`/discussions/${d.number}/comments?per_page=50`); - if (!comments) return 0; + // Walk every page until a short one (same class of bug as issue comments, + // owner fix 2026-11-07; the discussion variant missed it 2026-11-14): + // a single fixed page truncated long threads FOREVER — newer comments + // sat beyond it, were never imported, and the updated_at cursor advanced + // past them so they could never be retried (awesome-dsh-plugin/5795). let n = 0; - for (const gc of comments) { - if (await getCommentByUpstreamId(gc.id)) continue; - if ((gc.body ?? "").includes("")) continue; - const gcAuthor = gc.user?.login ?? "upstream"; - await ensureUser(gcAuthor, fromGithubBot(gcAuthor) ? "bot" : "human"); - const body = (gc.body ?? "") + "\n\n"; - const row = await postComment(local.id, body, gcAuthor, { - createdAt: gc.created_at, - updatedAt: gc.updated_at, - upstreamCommentId: gc.id, - }); - if (emit) void emitCommentEvent(this.project.id, local.id, row.id, this.origin); - n++; + for (let page = 1; page <= 50; page++) { + const comments = await this.fetchJson(`/discussions/${d.number}/comments?per_page=100&page=${page}`); + if (!comments || comments.length === 0) break; + for (const gc of comments) { + if (await getCommentByUpstreamId(gc.id)) continue; + if ((gc.body ?? "").includes("")) continue; + const gcAuthor = gc.user?.login ?? "upstream"; + await ensureUser(gcAuthor, fromGithubBot(gcAuthor) ? "bot" : "human"); + const body = (gc.body ?? "") + "\n\n"; + const row = await postComment(local.id, body, gcAuthor, { + createdAt: gc.created_at, + updatedAt: gc.updated_at, + upstreamCommentId: gc.id, + }); + if (emit) void emitCommentEvent(this.project.id, local.id, row.id, this.origin); + n++; + } + if (comments.length < 100) break; } return n; } From cde789d9a0bbd879013c3a29230c29aa926e6564 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 14 Nov 2026 12:41:26 +0800 Subject: [PATCH 2/2] test(web): regression test for discussion comment pagination (5795 loss class) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pagination fix in 0840916 shipped without a test pinning the new behavior; the 2026-11-07 issue-comment fix (c169a72) carried regression tests, the discussion variant missed them. - discussions-sync mock now slices per page/per_page (same helper as upstream-sync.test.ts) - new test: 100-comment backfill, then 20 new comments landing on page 2 in a live poll — all 120 imported, 20 comment webhooks emitted, cursor advances. Verified RED on the pre-fix code (only page 1 fetched). --- packages/web/tests/discussions-sync.test.ts | 44 ++++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/packages/web/tests/discussions-sync.test.ts b/packages/web/tests/discussions-sync.test.ts index 20c78d0..a9dec18 100644 --- a/packages/web/tests/discussions-sync.test.ts +++ b/packages/web/tests/discussions-sync.test.ts @@ -56,12 +56,19 @@ function mockFetch() { hookCalls.push({ url, body: JSON.parse(String(init?.body ?? "{}")) }); return new Response("{}", { status: 200 }); } + const slicePaged = (arr: T[]): T[] => { + const lm = url.match(/[?&](?:limit|per_page)=(\d+)/); + const limit = lm ? Number(lm[1]) : 100; + const pm = url.match(/[?&]page=(\d+)/); + const page = pm ? Number(pm[1]) : 1; + return arr.slice((page - 1) * limit, page * limit); + }; const perDiscussion = url.match(/\/discussions\/(\d+)\/comments/); if (perDiscussion) { - return Response.json(discussionComments[Number(perDiscussion[1])] ?? []); + return Response.json(slicePaged(discussionComments[Number(perDiscussion[1])] ?? [])); } if (/\/discussions(\?|$)/.test(url)) { - return Response.json(discussionList); + return Response.json(slicePaged(discussionList)); } if (/\/issues(\?|$)/.test(url)) { return Response.json(issueList); @@ -206,4 +213,37 @@ describe("discussions sync", () => { const local = await getIssueByUpstreamNumber(sync.project_id, 51, "discussion"); expect(local?.state).toBe("closed"); }); + + test("live poll walks every discussion comment page (thread > one page, 2026-11-14 discussion 5795 class)", async () => { + const { sync } = await setup(); + const ts = (day: number, i: number) => new Date(Date.UTC(2026, 9, day, 1, 0, i)).toISOString(); + mockFetch(); + // Round 1 (backfill): discussion 61 with 100 comments — all must land + discussionList = [discussion(61, { created_at: "2026-10-01T00:00:00Z", updated_at: "2026-10-02T00:00:00Z" })]; + discussionComments[61] = Array.from({ length: 100 }, (_, i) => + dcomment(6100 + i, 61, { created_at: ts(2, i), updated_at: ts(2, i) }) + ); + await (await makeSyncer(sync.project_id)).pollOnce(); + await settle(); + expect(await getCommentByUpstreamId(6199)).toBeTruthy(); + expect((await getUpstreamSync(sync.project_id))?.discussion_cursor).toBe("2026-10-02T00:00:00Z"); + + // Round 2 (live): 20 more comments land on page 2 and updated_at moves past + // the cursor. The old single-page fetch (per_page=50, page 1) saw only the + // already-imported comments, emitted no webhook, and still advanced the + // cursor — the 20 new comments were lost forever (discussion 5795). + discussionList = [discussion(61, { created_at: "2026-10-01T00:00:00Z", updated_at: "2026-10-05T00:00:00Z" })]; + discussionComments[61] = [ + ...discussionComments[61], + ...Array.from({ length: 20 }, (_, i) => dcomment(6200 + i, 61, { created_at: ts(5, i), updated_at: ts(5, i) })), + ]; + hookCalls = []; + await (await makeSyncer(sync.project_id)).pollOnce(); + await settle(); + + for (const id of [6200, 6209, 6210, 6219]) expect(await getCommentByUpstreamId(id)).toBeTruthy(); + const created = hookCalls.filter((c) => c.body.action === "created" && c.body.issue?.kind === "discussion"); + expect(created.length).toBe(20); + expect((await getUpstreamSync(sync.project_id))?.discussion_cursor).toBe("2026-10-05T00:00:00Z"); + }); });