Skip to content

Commit a9e9025

Browse files
committed
feat(api): window cursor walks led by a minute-rounded sort key
Walks sorted by toStartOfSecond/Minute/FiveMinutes/FifteenMinutes, with an exact timestamp part in the source expression, now use windows floored to the key's bucket and cut at the last row's bucket. These buckets divide 15 minutes, so they align to UTC in every current time zone. Other rounded keys keep a single window.
1 parent a476439 commit a9e9025

5 files changed

Lines changed: 261 additions & 73 deletions

File tree

‎packages/api/src/routers/external-api/__tests__/search.int.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -984,6 +984,76 @@ describe('External API v2 Search', () => {
984984
},
985985
);
986986

987+
it('pages a minute-rounded sort in order with minute-aligned windows and cuts', async () => {
988+
const minuteSource = await Source.create({
989+
kind: SourceKind.Log,
990+
team: logSource.team,
991+
from: logSource.from,
992+
defaultTableSelectExpression: 'Timestamp, SeverityText, Body',
993+
timestampValueExpression: 'toStartOfMinute(Timestamp), Timestamp',
994+
connection: logSource.connection,
995+
name: 'Logs (minute key)',
996+
});
997+
// 20 rows a minute for 30 minutes, so the walk crosses the first window's
998+
// boundary; Body order is unrelated to time within each minute.
999+
const base = DEFAULT_END_TIME - 30 * 60 * 1000;
1000+
await bulkInsertLogs(
1001+
Array.from({ length: 600 }, (_, i) => ({
1002+
ServiceName: 'cursor-minute-test',
1003+
Timestamp: new Date(base + i * 3000),
1004+
SeverityText: 'INFO',
1005+
Body: `row-${String((i * 37) % 600).padStart(3, '0')}`,
1006+
})),
1007+
);
1008+
1009+
const rows: { minute: number; body: string }[] = [];
1010+
const cursors: {
1011+
ws: string;
1012+
we: string;
1013+
o: number;
1014+
s: string;
1015+
e: string;
1016+
}[] = [];
1017+
let cursor: string | null | undefined;
1018+
for (let pages = 0; pages < 100; pages++) {
1019+
const res = await search({
1020+
...cursorBody(),
1021+
sourceId: minuteSource.id.toString(),
1022+
where: 'ServiceName:"cursor-minute-test"',
1023+
orderBy: 'toStartOfMinute(Timestamp) DESC, Body ASC',
1024+
maxResults: 50,
1025+
cursor: cursor ?? 'start',
1026+
}).expect(200);
1027+
for (const r of res.body.data) {
1028+
const minute = Math.floor(Date.parse(r.Timestamp) / 60_000);
1029+
rows.push({ minute, body: r.Body });
1030+
}
1031+
cursor = res.body.nextCursor;
1032+
if (cursor == null) break;
1033+
cursors.push(
1034+
JSON.parse(Buffer.from(cursor.split('.')[0], 'base64url').toString()),
1035+
);
1036+
}
1037+
1038+
expect(cursor).toBeNull();
1039+
expect(new Set(rows.map(r => r.body)).size).toBe(600);
1040+
rows.slice(1).forEach((r, i) => {
1041+
const prev = rows.at(i);
1042+
expect(prev).toBeDefined();
1043+
if (prev && r.minute === prev.minute)
1044+
expect(r.body > prev.body).toBe(true);
1045+
else if (prev) expect(r.minute < prev.minute).toBe(true);
1046+
});
1047+
for (const c of cursors) {
1048+
expect(c.o).toBeLessThanOrEqual(20);
1049+
for (const bound of [c.ws, c.we]) {
1050+
if (bound !== c.s && bound !== c.e) {
1051+
expect(Date.parse(bound) % 60_000).toBe(0);
1052+
}
1053+
}
1054+
}
1055+
});
1056+
9871057
it('rejects a select that uses the reserved cursor column name', async () => {
9881058
await search({
9891059
...cursorBody(),

‎packages/api/src/routers/external-api/v2/utils/__tests__/searchWindowCut.test.ts‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { cutWindowAtSecond } from '@/routers/external-api/v2/utils/searchWindows';
1+
import { cutWindowAtBucket } from '@/routers/external-api/v2/utils/searchWindows';
22

33
const window = {
44
ordinal: 0,
@@ -7,15 +7,16 @@ const window = {
77
};
88
const ms = (iso: string) => Date.parse(iso);
99

10-
describe('cutWindowAtSecond', () => {
10+
describe('cutWindowAtBucket', () => {
1111
it('ends a descending window just after the last row second', () => {
1212
expect(
13-
cutWindowAtSecond({
13+
cutWindowAtBucket({
1414
window,
1515
direction: 'DESC',
1616
offset: 0,
1717
pageLength: 2000,
18-
lastSecondMs: ms('2026-10-25T11:58:31.000Z'),
18+
lastBucketMs: ms('2026-10-25T11:58:31.000Z'),
19+
bucketMs: 1000,
1920
trailingRunLength: 12,
2021
}),
2122
).toEqual({
@@ -26,25 +27,27 @@ describe('cutWindowAtSecond', () => {
2627

2728
it('keeps the range end when the last row second reaches it', () => {
2829
expect(
29-
cutWindowAtSecond({
30+
cutWindowAtBucket({
3031
window,
3132
direction: 'DESC',
3233
offset: 0,
3334
pageLength: 2000,
34-
lastSecondMs: ms('2026-10-25T12:00:00.000Z'),
35+
lastBucketMs: ms('2026-10-25T12:00:00.000Z'),
36+
bucketMs: 1000,
3537
trailingRunLength: 2000,
3638
}),
3739
).toEqual({ window, offset: 2000 });
3840
});
3941

4042
it('starts an ascending window at the last row second', () => {
4143
expect(
42-
cutWindowAtSecond({
44+
cutWindowAtBucket({
4345
window,
4446
direction: 'ASC',
4547
offset: 0,
4648
pageLength: 2000,
47-
lastSecondMs: ms('2026-10-25T11:46:10.000Z'),
49+
lastBucketMs: ms('2026-10-25T11:46:10.000Z'),
50+
bucketMs: 1000,
4851
trailingRunLength: 7,
4952
}),
5053
).toEqual({
@@ -53,6 +56,23 @@ describe('cutWindowAtSecond', () => {
5356
});
5457
});
5558

59+
it.each([
60+
['DESC', { ...window, endTime: '2026-10-25T11:59:00.000Z' }],
61+
['ASC', { ...window, startTime: '2026-10-25T11:58:00.000Z' }],
62+
] as const)('cuts a %s window at a minute bucket', (direction, expected) => {
63+
expect(
64+
cutWindowAtBucket({
65+
window,
66+
direction,
67+
offset: 0,
68+
pageLength: 2000,
69+
lastBucketMs: ms('2026-10-25T11:58:00.000Z'),
70+
bucketMs: 60_000,
71+
trailingRunLength: 40,
72+
}),
73+
).toEqual({ window: expected, offset: 40 });
74+
});
75+
5676
it.each(['DESC', 'ASC'] as const)(
5777
'adds the page to the offset when the whole %s page was one second',
5878
direction => {
@@ -61,12 +81,13 @@ describe('cutWindowAtSecond', () => {
6181
? { ...window, endTime: '2026-10-25T11:58:32.000Z' }
6282
: { ...window, startTime: '2026-10-25T11:58:31.000Z' };
6383
expect(
64-
cutWindowAtSecond({
84+
cutWindowAtBucket({
6585
window: cut,
6686
direction,
6787
offset: 300,
6888
pageLength: 1000,
69-
lastSecondMs: ms('2026-10-25T11:58:31.000Z'),
89+
lastBucketMs: ms('2026-10-25T11:58:31.000Z'),
90+
bucketMs: 1000,
7091
trailingRunLength: 1000,
7192
}),
7293
).toEqual({ window: cut, offset: 1300 });

‎packages/api/src/routers/external-api/v2/utils/__tests__/searchWindows.test.ts‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ async function resolveSearchWindows(
3737
for (
3838
let w: typeof first.window | null = first.window;
3939
w != null;
40-
w = nextSearchWindow(w, range, first.direction)
40+
w = nextSearchWindow(w, range, first.direction, first.cut?.bucketMs)
4141
) {
4242
windows.push({
4343
startTime: new Date(w.startTime),
@@ -51,7 +51,12 @@ async function resolveSearchWindows(
5151

5252
// Reference windows: build every window, floor the boundaries between
5353
// them to the second, drop empties. Stepping must reproduce it exactly.
54-
function referenceWindows(start: Date, end: Date, direction: 'ASC' | 'DESC') {
54+
function referenceWindows(
55+
start: Date,
56+
end: Date,
57+
direction: 'ASC' | 'DESC',
58+
alignMs = 1000,
59+
) {
5560
const windows =
5661
direction === 'ASC'
5762
? generateTimeWindowsAscending(start, end)
@@ -60,7 +65,7 @@ function referenceWindows(start: Date, end: Date, direction: 'ASC' | 'DESC') {
6065
const snap = (t: Date) =>
6166
t.getTime() === start.getTime() || t.getTime() === end.getTime()
6267
? t.getTime()
63-
: Math.max(Math.floor(t.getTime() / 1000) * 1000, start.getTime());
68+
: Math.max(Math.floor(t.getTime() / alignMs) * alignMs, start.getTime());
6469
return windows
6570
.map(w => [snap(w.startTime), snap(w.endTime)])
6671
.filter(([a, b]) => a < b)
@@ -225,9 +230,25 @@ describe('search window stepping', () => {
225230
},
226231
);
227232

228-
it.each(['DESC', 'ASC'] as const)(
229-
'steps through exactly the reference windows (%s)',
230-
async direction => {
233+
it.each([
234+
['DESC', 'Timestamp', 'Timestamp', 1000],
235+
['ASC', 'Timestamp', 'Timestamp', 1000],
236+
[
237+
'DESC',
238+
'toStartOfMinute(Timestamp), Timestamp',
239+
'toStartOfMinute(Timestamp)',
240+
60_000,
241+
],
242+
[
243+
'ASC',
244+
'toStartOfMinute(Timestamp), Timestamp',
245+
'toStartOfMinute(Timestamp)',
246+
60_000,
247+
],
248+
] as const)(
249+
'steps through exactly the reference windows (%s, %s)',
250+
async (direction, timestampValueExpression, sortKey, alignMs) => {
251+
const walkSource = { timestampValueExpression };
231252
// Deterministic pseudo-random ranges: 0 to 30 days, millisecond precision.
232253
let seed = 42;
233254
const rand = () =>
@@ -238,16 +259,16 @@ describe('search window stepping', () => {
238259
let cumulative = 0;
239260
for (const size of [...DEFAULT_TIME_WINDOWS_SECONDS, 86_400, 86_400]) {
240261
cumulative += size * 1000;
241-
for (const delta of [-1300, -300, 0, 300, 1300]) {
262+
for (const delta of [-61_300, -1300, -300, 0, 300, 1300, 61_300]) {
242263
const end = new Date(edgeStart.getTime() + cumulative + delta);
243264
const walked = await resolveSearchWindows(
244-
source,
245-
`Timestamp ${direction}`,
265+
walkSource,
266+
`${sortKey} ${direction}`,
246267
edgeStart,
247268
end,
248269
);
249270
expect(walked.map(w => [w.startTime, w.endTime])).toEqual(
250-
referenceWindows(edgeStart, end, direction),
271+
referenceWindows(edgeStart, end, direction, alignMs),
251272
);
252273
}
253274
}
@@ -259,13 +280,13 @@ describe('search window stepping', () => {
259280
start.getTime() + Math.floor(rand() * 30 * 86_400_000),
260281
);
261282
const walked = await resolveSearchWindows(
262-
source,
263-
`Timestamp ${direction}`,
283+
walkSource,
284+
`${sortKey} ${direction}`,
264285
start,
265286
end,
266287
);
267288
expect(walked.map(w => [w.startTime, w.endTime])).toEqual(
268-
referenceWindows(start, end, direction),
289+
referenceWindows(start, end, direction, alignMs),
269290
);
270291
}
271292
},
@@ -318,20 +339,47 @@ describe('search window stepping', () => {
318339
},
319340
);
320341
expect(first.windowed).toBe(true);
321-
expect(first.cutPart).toBe(cutPart);
342+
expect(first.cut).toEqual({ expression: cutPart, bucketMs: 1000 });
322343
},
323344
);
324345

325346
it.each([
347+
['toStartOfSecond(Timestamp)', 1000],
348+
['toStartOfMinute(Timestamp)', 60_000],
349+
['toStartOfFiveMinutes(Timestamp)', 300_000],
350+
['toStartOfFiveMinute(Timestamp)', 300_000],
351+
['toStartOfFifteenMinutes(Timestamp)', 900_000],
352+
])(
353+
'cuts on a rounded leading key %s in %ims buckets',
354+
async (key, bucketMs) => {
355+
const first = await resolveFirstSearchWindow(
356+
{ timestampValueExpression: `${key}, Timestamp` },
357+
`${key} DESC, Body`,
358+
START,
359+
END,
360+
{ columnType: async () => 'DateTime64(9)' },
361+
);
362+
expect(first.windowed).toBe(true);
363+
expect(first.cut).toEqual({ expression: key, bucketMs });
364+
},
365+
);
366+
367+
it.each([
368+
// Ten-minute buckets don't align to UTC in +x:45 time zones.
326369
[
327-
'toStartOfMinute(Timestamp), Timestamp',
328-
'toStartOfMinute(Timestamp) DESC, Body',
370+
'toStartOfTenMinutes(Timestamp), Timestamp',
371+
'toStartOfTenMinutes(Timestamp) DESC, Body',
372+
],
373+
[
374+
'toStartOfHour(Timestamp), Timestamp',
375+
'toStartOfHour(Timestamp) DESC, Body',
329376
],
330377
['EventDate, EventTime', 'EventDate DESC, Body'],
378+
// Without an exact part, rounded bounds are inclusive on both ends.
379+
['toStartOfMinute(Timestamp)', 'toStartOfMinute(Timestamp) DESC, Body'],
331380
])(
332-
'uses one window for %s ordered by a rounded key (%s)',
381+
'uses one window for %s ordered by %s',
333382
async (timestampValueExpression, orderBy) => {
334-
// A window boundary inside one rounded key would break the requested order.
335383
const first = await resolveFirstSearchWindow(
336384
{ timestampValueExpression },
337385
orderBy,
@@ -343,7 +391,7 @@ describe('search window stepping', () => {
343391
},
344392
);
345393
expect(first.windowed).toBe(false);
346-
expect(first.cutPart).toBeNull();
394+
expect(first.cut).toBeNull();
347395
},
348396
);
349397

0 commit comments

Comments
 (0)