Skip to content

Commit a476439

Browse files
committed
perf(api): sign search cursors and cut windows at the last row's second
Cursors are HMAC-signed with EXPRESS_SESSION_SECRET and carry the current window's bounds, so the next window is computed directly and the range cap is gone. Each full page cuts its window at the last row's second, keeping OFFSET small in dense ranges. Only walks led by an exact timestamp key use windows; others read one window with the 10,000 offset cap. Cursor walks force the overflow modes to throw instead of truncating.
1 parent 61b9dd9 commit a476439

16 files changed

Lines changed: 1098 additions & 311 deletions

File tree

‎.changeset/search-cursor-pagination.md‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,4 @@
33
'@hyperdx/common-utils': patch
44
---
55

6-
Add cursor pagination to the external API v2 search endpoint. Pass the
7-
`nextCursor` from a response back as `cursor` to fetch the next page; pages are
8-
scoped to a time window instead of a growing `OFFSET`, so page cost no longer
9-
grows with depth. The existing `offset` parameter is unchanged.
10-
11-
Keep requesting pages until `nextCursor` is `null` — a short page does not mean
12-
the results are exhausted.
6+
Add cursor pagination to the external API v2 search endpoint: send `cursor: "start"`, then pass back each `nextCursor` until it is `null`. Cursors are signed with `EXPRESS_SESSION_SECRET`: replicas need a shared value, and rotating it invalidates outstanding cursors.

‎packages/api/openapi.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4818,7 +4818,7 @@
48184818
"type": "string",
48194819
"maxLength": 1024,
48204820
"example": "start",
4821-
"description": "Send the literal string \"start\" to begin cursor paging, then pass\nback the nextCursor from each response. When set, offset is\nignored; omit cursor entirely to use offset-based paging.\nCursor paging supports time ranges of up to 366 days. As with\noffset, rows that share a timestamp can be skipped or repeated\nbetween pages unless orderBy breaks the tie.\n\nKeep requesting pages until nextCursor is null. A page that is\nempty or returns fewer rows than maxResults does NOT mean the\nresults are exhausted.\n"
4821+
"description": "Send the literal string \"start\" to begin cursor paging, then pass\nback the nextCursor from each response. When set, offset is\nignored; omit cursor entirely to use offset-based paging.\nAs with offset, rows that share a timestamp can be skipped or\nrepeated between pages unless orderBy breaks the tie.\n\nKeep requesting pages until nextCursor is null. A page that is\nempty or returns fewer rows than maxResults does NOT mean the\nresults are exhausted.\n"
48224822
}
48234823
}
48244824
},

‎packages/api/src/routers/external-api/__tests__/search.int.test.ts‎

Lines changed: 303 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { QuerySettings } from '@hyperdx/common-utils/dist/types';
12
import { SourceKind } from '@hyperdx/common-utils/dist/types';
23
import mongoose from 'mongoose';
34
import request from 'supertest';
@@ -18,6 +19,7 @@ import {
1819
encodeSearchCursor,
1920
fingerprintQuery,
2021
} from '@/routers/external-api/v2/utils/searchCursor';
22+
import logger from '@/utils/logger';
2123

2224
const DEFAULT_END_TIME = Math.floor(Date.now() / 60000) * 60000;
2325
const DEFAULT_START_TIME = DEFAULT_END_TIME - 3600 * 1000;
@@ -582,6 +584,9 @@ describe('External API v2 Search', () => {
582584
]);
583585
};
584586

587+
const cursorOffset = (cursor: string) =>
588+
JSON.parse(Buffer.from(cursor.split('.')[0], 'base64url').toString()).o;
589+
585590
const cursorBody = () => ({
586591
sourceId: logSource.id.toString(),
587592
startTime: iso(DEFAULT_START_TIME),
@@ -703,21 +708,316 @@ describe('External API v2 Search', () => {
703708
},
704709
);
705710

706-
it('rejects a cursor walk over a range longer than a year', async () => {
707-
await search({
711+
it('starts a cursor walk over a range longer than a year', async () => {
712+
await seedCursorRows();
713+
714+
const res = await search({
708715
...cursorBody(),
709716
startTime: iso(DEFAULT_END_TIME - 400 * 24 * 60 * 60 * 1000),
710717
cursor: 'start',
718+
}).expect(200);
719+
expect(res.body.data).toHaveLength(5);
720+
expect(res.body.nextCursor).toBeTruthy();
721+
});
722+
723+
it('pages past 10,000 rows in a range shorter than the first window', async () => {
724+
const rangeStart = DEFAULT_END_TIME - 10 * 60 * 1000;
725+
await bulkInsertLogs(
726+
Array.from({ length: 10_050 }, (_, i) => ({
727+
ServiceName: 'cursor-dense-test',
728+
Timestamp: new Date(rangeStart + i * 50),
729+
SeverityText: 'INFO',
730+
Body: `dense-${i}`,
731+
})),
732+
);
733+
const body = {
734+
...cursorBody(),
735+
startTime: iso(rangeStart),
736+
where: 'ServiceName:"cursor-dense-test"',
737+
maxResults: 2000,
738+
};
739+
740+
const seen = new Set<string>();
741+
const offsets: number[] = [];
742+
let cursor: string | null | undefined;
743+
for (let pages = 0; pages < 20; pages++) {
744+
const res = await search({ ...body, cursor: cursor ?? 'start' }).expect(
745+
200,
746+
);
747+
res.body.data.forEach((r: Record<string, string>) => seen.add(r.Body));
748+
cursor = res.body.nextCursor;
749+
if (cursor == null) break;
750+
offsets.push(cursorOffset(cursor));
751+
}
752+
expect(cursor).toBeNull();
753+
expect(seen.size).toBe(10_050);
754+
// Rows are 50ms apart, so a second holds 20 of them.
755+
expect(Math.max(...offsets)).toBeLessThanOrEqual(20);
756+
});
757+
758+
it.each(['Timestamp DESC', 'Timestamp ASC'])(
759+
'returns every row once when one second holds more rows than a page (%s)',
760+
async orderBy => {
761+
const second = DEFAULT_END_TIME - 5 * 60 * 1000;
762+
await bulkInsertLogs(
763+
Array.from({ length: 900 }, (_, i) => ({
764+
ServiceName: 'cursor-burst-test',
765+
Timestamp: new Date(second + i),
766+
SeverityText: 'INFO',
767+
Body: `burst-${i}`,
768+
})),
769+
);
770+
const body = {
771+
...cursorBody(),
772+
where: 'ServiceName:"cursor-burst-test"',
773+
orderBy,
774+
maxResults: 300,
775+
};
776+
777+
const seen: string[] = [];
778+
let cursor: string | null | undefined;
779+
for (let pages = 0; pages < 20; pages++) {
780+
const res = await search({ ...body, cursor: cursor ?? 'start' }).expect(
781+
200,
782+
);
783+
seen.push(...res.body.data.map((r: Record<string, string>) => r.Body));
784+
cursor = res.body.nextCursor;
785+
if (cursor == null) break;
786+
}
787+
expect(cursor).toBeNull();
788+
expect(seen).toHaveLength(900);
789+
expect(new Set(seen).size).toBe(900);
790+
},
791+
);
792+
793+
it('walks every row exactly once in ascending order', async () => {
794+
await seedCursorRows();
795+
// seedCursorRows puts four rows on one Timestamp; Body breaks the tie.
796+
797+
const seen: string[] = [];
798+
let cursor: string | null | undefined;
799+
for (let pages = 0; pages < 50; pages++) {
800+
const res = await search({
801+
...cursorBody(),
802+
orderBy: 'Timestamp ASC, Body ASC',
803+
cursor: cursor ?? 'start',
804+
}).expect(200);
805+
seen.push(...res.body.data.map((r: Record<string, string>) => r.Body));
806+
cursor = res.body.nextCursor;
807+
if (cursor == null) break;
808+
}
809+
expect(cursor).toBeNull();
810+
expect(seen).toHaveLength(43);
811+
expect(new Set(seen).size).toBe(43);
812+
});
813+
814+
it.each([
815+
['omitted', {}],
816+
['without a timestamp', { select: 'Body' }],
817+
])(
818+
'never returns the cursor timestamp column when select is %s',
819+
async (_label, selectField) => {
820+
await seedCursorRows();
821+
const { select: _omit, ...rest } = cursorBody();
822+
const res = await search({
823+
...rest,
824+
...selectField,
825+
cursor: 'start',
826+
}).expect(200);
827+
expect(res.body.data.length).toBeGreaterThan(0);
828+
for (const row of res.body.data) {
829+
expect(row).not.toHaveProperty('__hdx_cursor_ts');
830+
}
831+
},
832+
);
833+
834+
const createLimitedSource = (querySettings: QuerySettings) =>
835+
Source.create({
836+
kind: SourceKind.Log,
837+
team: logSource.team,
838+
from: logSource.from,
839+
defaultTableSelectExpression: 'Timestamp, SeverityText, Body',
840+
timestampValueExpression: 'Timestamp',
841+
connection: logSource.connection,
842+
name: 'Logs (limited)',
843+
querySettings,
844+
});
845+
846+
it('fails a cursor walk instead of truncating when source settings say break', async () => {
847+
await seedCursorRows();
848+
const limited = await createLimitedSource([
849+
{ setting: 'max_rows_to_read', value: '1' },
850+
{ setting: 'read_overflow_mode', value: 'break' },
851+
]);
852+
const errorSpy = jest.spyOn(logger, 'error');
853+
854+
const { maxResults: _max, ...body } = cursorBody();
855+
// Offset paging keeps the source's 'break' setting; a cursor walk overrides
856+
// it and fails instead of truncating.
857+
await search({ ...body, sourceId: limited.id.toString() }).expect(200);
858+
expect(errorSpy).not.toHaveBeenCalled();
859+
const res = await search({
860+
...body,
861+
sourceId: limited.id.toString(),
862+
cursor: 'start',
863+
});
864+
expect(res.status).toBe(500);
865+
expect(errorSpy).toHaveBeenCalledWith(
866+
expect.objectContaining({ chType: 'TOO_MANY_ROWS' }),
867+
'[search] ClickHouse query error',
868+
);
869+
errorSpy.mockRestore();
870+
});
871+
872+
it.each([5, 100])(
873+
'walks every row when maxResults %i is at or above the source max_result_rows',
874+
async maxResults => {
875+
await seedCursorRows();
876+
const limited = await createLimitedSource([
877+
{ setting: 'max_result_rows', value: '5' },
878+
{ setting: 'result_overflow_mode', value: 'break' },
879+
]);
880+
881+
const seen: string[] = [];
882+
let cursor: string | null | undefined;
883+
for (let pages = 0; pages < 50; pages++) {
884+
const res = await search({
885+
...cursorBody(),
886+
sourceId: limited.id.toString(),
887+
maxResults,
888+
cursor: cursor ?? 'start',
889+
}).expect(200);
890+
expect(res.body.data.length).toBeLessThanOrEqual(5);
891+
seen.push(...res.body.data.map((r: Record<string, string>) => r.Body));
892+
cursor = res.body.nextCursor;
893+
if (cursor == null) break;
894+
}
895+
896+
expect(cursor).toBeNull();
897+
expect(seen).toHaveLength(43);
898+
expect(new Set(seen).size).toBe(43);
899+
},
900+
);
901+
902+
it('keeps a DISTINCT select distinct during a cursor walk', async () => {
903+
const now = DEFAULT_END_TIME - 2 * 60 * 1000;
904+
await bulkInsertLogs(
905+
Array.from({ length: 30 }, (_, i) => ({
906+
ServiceName: 'cursor-distinct-test',
907+
// Half the rows are older than the first 15m window's start.
908+
Timestamp: new Date(now - (i < 15 ? i : 30 * 60 + i) * 1000),
909+
SeverityText: 'INFO',
910+
Body: `dup-${i % 3}`,
911+
})),
912+
);
913+
914+
const seen: string[] = [];
915+
let cursor: string | null | undefined;
916+
for (let pages = 0; pages < 20; pages++) {
917+
const res = await search({
918+
...cursorBody(),
919+
where: 'ServiceName:"cursor-distinct-test"',
920+
select: 'DISTINCT Body',
921+
maxResults: 2,
922+
cursor: cursor ?? 'start',
923+
}).expect(200);
924+
seen.push(...res.body.data.map((r: Record<string, string>) => r.Body));
925+
cursor = res.body.nextCursor;
926+
if (cursor == null) break;
927+
}
928+
expect(seen.sort()).toEqual(['dup-0', 'dup-1', 'dup-2']);
929+
});
930+
931+
it.each([
932+
[
933+
'a rounded leading key',
934+
'toStartOfMinute(Timestamp), Timestamp',
935+
'toStartOfMinute(Timestamp) DESC, Body ASC',
936+
],
937+
[
938+
'an exact leading key on a compound expression',
939+
'toDateTime(Timestamp), Timestamp',
940+
'Timestamp DESC',
941+
],
942+
])(
943+
'returns every row once when the walk is ordered by %s',
944+
async (_label, timestampValueExpression, orderBy) => {
945+
const compound = await Source.create({
946+
kind: SourceKind.Log,
947+
team: logSource.team,
948+
from: logSource.from,
949+
defaultTableSelectExpression: 'Timestamp, SeverityText, Body',
950+
timestampValueExpression,
951+
connection: logSource.connection,
952+
name: 'Logs (compound)',
953+
});
954+
const minute =
955+
Math.floor((DEFAULT_END_TIME - 3 * 60 * 1000) / 60_000) * 60_000;
956+
// Body order is unrelated to time, so a cut on the wrong key loses rows.
957+
await bulkInsertLogs(
958+
Array.from({ length: 120 }, (_, i) => ({
959+
ServiceName: 'cursor-order-test',
960+
Timestamp: new Date(minute + i * 400),
961+
SeverityText: 'INFO',
962+
Body: `row-${String((i * 37) % 120).padStart(3, '0')}`,
963+
})),
964+
);
965+
966+
const seen: string[] = [];
967+
let cursor: string | null | undefined;
968+
for (let pages = 0; pages < 50; pages++) {
969+
const res = await search({
970+
...cursorBody(),
971+
sourceId: compound.id.toString(),
972+
where: 'ServiceName:"cursor-order-test"',
973+
orderBy,
974+
maxResults: 10,
975+
cursor: cursor ?? 'start',
976+
}).expect(200);
977+
seen.push(...res.body.data.map((r: Record<string, string>) => r.Body));
978+
cursor = res.body.nextCursor;
979+
if (cursor == null) break;
980+
}
981+
expect(cursor).toBeNull();
982+
expect(seen).toHaveLength(120);
983+
expect(new Set(seen).size).toBe(120);
984+
},
985+
);
986+
987+
it('rejects a select that uses the reserved cursor column name', async () => {
988+
await search({
989+
...cursorBody(),
990+
select: 'Timestamp AS __hdx_cursor_ts',
991+
cursor: 'start',
711992
}).expect(400);
712993
});
713994

995+
it('rejects a cursor whose contents were edited', async () => {
996+
await seedCursorRows();
997+
998+
const first = await search({ ...cursorBody(), cursor: 'start' }).expect(
999+
200,
1000+
);
1001+
const [payload, signature] = String(first.body.nextCursor).split('.');
1002+
const edited = JSON.parse(Buffer.from(payload, 'base64url').toString());
1003+
edited.o += 1;
1004+
const forged = `${Buffer.from(JSON.stringify(edited)).toString('base64url')}.${signature}`;
1005+
1006+
const res = await search({ ...cursorBody(), cursor: forged }).expect(400);
1007+
expect(res.body.message).toMatch(/invalid or expired/);
1008+
});
1009+
7141010
const forgeCursor = (
7151011
body: ReturnType<typeof cursorBody> & { orderBy: string },
7161012
offset: number,
7171013
) =>
7181014
encodeSearchCursor(
7191015
{
720-
windowIndex: 0,
1016+
window: {
1017+
ordinal: 0,
1018+
startTime: iso(DEFAULT_START_TIME),
1019+
endTime: iso(DEFAULT_END_TIME),
1020+
},
7211021
offset,
7221022
startTime: iso(DEFAULT_START_TIME),
7231023
endTime: iso(DEFAULT_END_TIME),

0 commit comments

Comments
 (0)