From dd69d740465a04578bf5af35c09a218204edc40c Mon Sep 17 00:00:00 2001 From: Goransh walia Date: Mon, 9 Nov 2026 16:13:42 +0530 Subject: [PATCH] fix: debt: SyncScheduler request pump can strand queued requests (#102) --- net/sync-scheduler.ts | 189 ++++++++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 72 deletions(-) diff --git a/net/sync-scheduler.ts b/net/sync-scheduler.ts index 67385ce9..d39f1ef3 100644 --- a/net/sync-scheduler.ts +++ b/net/sync-scheduler.ts @@ -77,6 +77,7 @@ export enum SyncPriority { export class SyncScheduler { private readonly _syncFreqAvg: MovingAverage; private _pendingRequests: Map; + private _inFlightRequests = new Set(); private _intervalId: ReturnType | undefined; private _fetchInProgress = false; private _closed = false; @@ -114,6 +115,10 @@ export class SyncScheduler { } } this._pendingRequests.clear(); + for (const req of this._inFlightRequests) { + req.reject(serviceUnavailable()); + } + this._inFlightRequests.clear(); } send( @@ -163,93 +168,133 @@ export class SyncScheduler { if (!pendingRequests.length) { return; } - const reqArr = pendingRequests.map((p) => ({ - ...p.req, - msg: JSONCyclicalEncoder.serialize(p.req.msg), - })); - let respText: string | undefined; + this._fetchInProgress = true; + for (const req of pendingRequests) { + this._inFlightRequests.add(req); + } + try { - this._fetchInProgress = true; - const start = performance.now(); - const resp = await sendJSONToURL( - this.url + '/batch-sync', - this.trustPool.currentSession, - reqArr, - this.orgId, - this.syncConfig.timeoutMs, - ); - respText = resp.status === 200 ? await resp.text() : undefined; - - const syncDurationMs = performance.now() - start; - this._syncFreqAvg.addValue(syncDurationMs); - if (randomInt(0, 20) === 0) { + let reqArr: ReadonlyJSONObject[]; + try { + reqArr = pendingRequests.map((p) => ({ + ...p.req, + msg: JSONCyclicalEncoder.serialize(p.req.msg), + })); + } catch (e: unknown) { log({ - severity: 'METRIC', - name: 'PeerResponseTime', - value: syncDurationMs, - unit: 'Milliseconds', - url: this.url, + severity: 'INFO', + error: 'SerializeError', + message: (e as Error).message, + trace: (e as Error).stack, }); + pendingRequests.forEach((r) => r.reject(serviceUnavailable())); + return; } - } catch (e: unknown) { - log({ - severity: 'INFO', - error: 'FetchError', - message: (e as Error).message, - trace: (e as Error).stack, - url: this.url, - }); - this._fetchInProgress = false; - } - if (!respText) { - this._fetchInProgress = false; - pendingRequests.forEach((r) => r.reject(serviceUnavailable())); - return; - } - try { - const json = JSON.parse(respText) as ReadonlyJSONArray; - assert(json instanceof Array && json.length === pendingRequests.length); - for (const req of pendingRequests) { - let found = false; - for (const resp of json as ReadonlyJSONObject[]) { - if (resp.path === req.req.path) { - const decoder = JSONCyclicalDecoder.get( - resp.res as ReadonlyJSONObject, - ); - const syncResp = await SyncMessage.decodeAsync( - { - decoder: decoder, - orgId: this.orgId, - }, - this.registry, - ); - decoder.finalize(); - req.resolve(syncResp); - found = true; - break; - } - } - if (!found) { - req.reject(serviceUnavailable()); + let respText: string | undefined; + try { + const start = performance.now(); + const resp = await sendJSONToURL( + this.url + '/batch-sync', + this.trustPool.currentSession, + reqArr, + this.orgId, + this.syncConfig.timeoutMs, + ); + respText = resp.status === 200 ? await resp.text() : undefined; + + const syncDurationMs = performance.now() - start; + this._syncFreqAvg.addValue(syncDurationMs); + if (randomInt(0, 20) === 0) { + log({ + severity: 'METRIC', + name: 'PeerResponseTime', + value: syncDurationMs, + unit: 'Milliseconds', + url: this.url, + }); } - await sleep(5); - } - } catch (e: unknown) { - if (!(e instanceof ServerError && e.code === Code.ServiceUnavailable)) { + } catch (e: unknown) { log({ severity: 'INFO', - error: 'SerializeError', - value: respText, + error: 'FetchError', message: (e as Error).message, trace: (e as Error).stack, + url: this.url, }); } + + if (!respText) { + pendingRequests.forEach((r) => r.reject(serviceUnavailable())); + return; + } + + try { + const json = JSON.parse(respText) as ReadonlyJSONArray; + assert(json instanceof Array && json.length === pendingRequests.length); + for (const req of pendingRequests) { + let found = false; + try { + for (const resp of json as ReadonlyJSONObject[]) { + if (resp.path === req.req.path) { + const decoder = JSONCyclicalDecoder.get( + resp.res as ReadonlyJSONObject, + ); + const syncResp = await SyncMessage.decodeAsync( + { + decoder: decoder, + orgId: this.orgId, + }, + this.registry, + ); + decoder.finalize(); + req.resolve(syncResp); + found = true; + break; + } + } + if (!found) { + req.reject(serviceUnavailable()); + } + } catch (e: unknown) { + if ( + !(e instanceof ServerError && e.code === Code.ServiceUnavailable) + ) { + log({ + severity: 'INFO', + error: 'SerializeError', + value: respText, + message: (e as Error).message, + trace: (e as Error).stack, + }); + } + req.reject(serviceUnavailable()); + } + await sleep(5); + } + } catch (e: unknown) { + if ( + !(e instanceof ServerError && e.code === Code.ServiceUnavailable) + ) { + log({ + severity: 'INFO', + error: 'SerializeError', + value: respText, + message: (e as Error).message, + trace: (e as Error).stack, + }); + } + pendingRequests.forEach((r) => r.reject(serviceUnavailable())); + return; + } + } catch (e: unknown) { pendingRequests.forEach((r) => r.reject(serviceUnavailable())); - return; } finally { + for (const req of pendingRequests) { + this._inFlightRequests.delete(req); + } this._fetchInProgress = false; } } -} +} \ No newline at end of file