Skip to content

Commit 774ffef

Browse files
committed
fix: isolate usage-stats headers, exit once, stop telemetry after both servers
1 parent 8f9a4f5 commit 774ffef

10 files changed

Lines changed: 373 additions & 66 deletions

File tree

‎packages/api/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"@opentelemetry/api": "~1.9.1",
3030
"@opentelemetry/api-logs": "^0.223.0",
3131
"@opentelemetry/core": "^2.12.0",
32-
"@opentelemetry/exporter-logs-otlp-http": "^0.223.0",
32+
"@opentelemetry/otlp-transformer": "^0.223.0",
3333
"@opentelemetry/resources": "^2.12.0",
3434
"@opentelemetry/sdk-logs": "^0.223.0",
3535
"@slack/webhook": "^7.0.0",

‎packages/api/src/server.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { connectDBWithRetry, mongooseConnection } from '@/models';
1111
import opampApp from '@/opamp/app';
1212
import { setupTeamDefaults } from '@/setupDefaults';
1313
import logger from '@/utils/logger';
14+
import { runOnceAfterAll } from '@/utils/runOnceAfterAll';
1415
import { verifySessionSecret } from '@/utils/sessionSecret';
1516
import { verifyTokenEncryption } from '@/utils/tokenEncryption';
1617

@@ -72,7 +73,14 @@ export default class Server {
7273
});
7374

7475
if (this.shouldHandleGracefulShutdown) {
75-
[this.appServer, this.opampServer].forEach(server => {
76+
const servers = [this.appServer, this.opampServer];
77+
// Each server's forceExit calls process.exit() once its own onShutdown
78+
// settles, so both wait here: telemetry stops once, after the slower
79+
// server has drained, and neither exits before then.
80+
const shutdownTelemetryOnce = runOnceAfterAll(servers.length, () =>
81+
shutdownTelemetry(),
82+
);
83+
servers.forEach(server => {
7684
gracefulShutdown(server, {
7785
signals: 'SIGINT SIGTERM',
7886
timeout: 10000, // 10 secs
@@ -91,7 +99,7 @@ export default class Server {
9199
// forceExit calls process.exit(), which skips the SDK's own
92100
// exit-time flush. Before init (no key set) this resolves at
93101
// once.
94-
await shutdownTelemetry();
102+
await shutdownTelemetryOnce();
95103
}
96104
},
97105
});
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
const mockShutdown = jest.fn<Promise<void>, []>();
2+
const mockCronStop = jest.fn();
3+
const mockCronFrom = jest.fn((_options: unknown) => ({ stop: mockCronStop }));
4+
const mockConfig = {
5+
CODE_VERSION: 'test',
6+
RUN_SCHEDULED_TASKS_EXTERNALLY: false,
7+
};
8+
9+
jest.mock('@clickhouse/clickstack-node', () => ({
10+
shutdown: () => mockShutdown(),
11+
}));
12+
jest.mock('cron', () => ({
13+
CronJob: { from: (options: unknown) => mockCronFrom(options) },
14+
}));
15+
jest.mock('@/config', () => mockConfig);
16+
jest.mock('@/utils/logger', () => ({
17+
__esModule: true,
18+
default: { info: jest.fn(), warn: jest.fn(), error: jest.fn() },
19+
}));
20+
jest.mock('@/tasks/checkAlerts', () => ({
21+
__esModule: true,
22+
default: jest.fn(),
23+
}));
24+
jest.mock('@/tasks/provisionDashboards', () => ({
25+
__esModule: true,
26+
default: jest.fn(),
27+
}));
28+
29+
const flushPromises = () => new Promise(resolve => setImmediate(resolve));
30+
31+
describe('tasks entry point', () => {
32+
const originalArgv = process.argv;
33+
// Jest's process copy forwards exitCode to the real process.
34+
const originalExitCode = process.exitCode;
35+
let processOn: jest.SpyInstance;
36+
let processExit: jest.SpyInstance;
37+
let finishShutdown: () => void;
38+
39+
const loadTasks = () => {
40+
process.argv = ['node', 'tasks', 'ping-pong'];
41+
jest.isolateModules(() => {
42+
jest.requireActual('@/tasks/index');
43+
});
44+
};
45+
46+
const handlerFor = (event: string) => {
47+
const call = processOn.mock.calls.find(([name]) => name === event);
48+
if (!call) {
49+
throw new Error(`No ${event} handler registered`);
50+
}
51+
return call[1];
52+
};
53+
54+
beforeEach(() => {
55+
jest.clearAllMocks();
56+
mockShutdown.mockReturnValue(
57+
new Promise(resolve => {
58+
finishShutdown = resolve;
59+
}),
60+
);
61+
processOn = jest.spyOn(process, 'on').mockReturnValue(process);
62+
processExit = jest.spyOn(process, 'exit').mockImplementation();
63+
jest.spyOn(console, 'log').mockImplementation();
64+
});
65+
66+
afterEach(() => {
67+
jest.restoreAllMocks();
68+
process.argv = originalArgv;
69+
process.exitCode = originalExitCode;
70+
});
71+
72+
it('stops the cron job and shuts down once, exiting with the first code', async () => {
73+
mockConfig.RUN_SCHEDULED_TASKS_EXTERNALLY = false;
74+
loadTasks();
75+
expect(mockCronFrom).toHaveBeenCalledTimes(1);
76+
77+
handlerFor('uncaughtException')(new Error('first'));
78+
handlerFor('unhandledRejection')(new Error('second'));
79+
handlerFor('uncaughtException')(new Error('third'));
80+
81+
expect(mockCronStop).toHaveBeenCalledTimes(1);
82+
expect(mockShutdown).toHaveBeenCalledTimes(1);
83+
expect(mockCronStop.mock.invocationCallOrder[0]).toBeLessThan(
84+
mockShutdown.mock.invocationCallOrder[0],
85+
);
86+
expect(process.exitCode).toBe(1);
87+
expect(processExit).not.toHaveBeenCalled();
88+
89+
finishShutdown();
90+
await flushPromises();
91+
92+
expect(processExit).toHaveBeenCalledTimes(1);
93+
expect(processExit).toHaveBeenCalledWith(1);
94+
});
95+
96+
it('keeps exit code 0 when a rejection arrives during the one-shot shutdown', async () => {
97+
mockConfig.RUN_SCHEDULED_TASKS_EXTERNALLY = true;
98+
loadTasks();
99+
await flushPromises();
100+
expect(mockCronFrom).not.toHaveBeenCalled();
101+
expect(mockShutdown).toHaveBeenCalledTimes(1);
102+
103+
handlerFor('unhandledRejection')(new Error('late'));
104+
expect(mockShutdown).toHaveBeenCalledTimes(1);
105+
106+
finishShutdown();
107+
await flushPromises();
108+
109+
expect(processExit).toHaveBeenCalledTimes(1);
110+
expect(processExit).toHaveBeenCalledWith(0);
111+
});
112+
});
Lines changed: 91 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,47 @@
11
import { context, ROOT_CONTEXT, trace, TraceFlags } from '@opentelemetry/api';
22
import { SeverityNumber } from '@opentelemetry/api-logs';
3-
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
4-
import { InMemoryLogRecordExporter } from '@opentelemetry/sdk-logs';
53
import os from 'os';
64

75
import { sendUsageStats } from '@/tasks/usageStats';
86

9-
const mockExporter = new InMemoryLogRecordExporter();
7+
const USAGE_STATS_URL = 'https://in-otel.hyperdx.io/v1/logs';
8+
const USAGE_STATS_API_KEY = '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0';
109

11-
jest.mock('@opentelemetry/exporter-logs-otlp-http', () => ({
12-
OTLPLogExporter: jest.fn(() => mockExporter),
13-
}));
10+
// jest.setup.ts replaces the global fetch with a jest.fn.
11+
const mockFetch = jest.mocked(fetch);
12+
13+
const sentRequest = async () => {
14+
expect(mockFetch).toHaveBeenCalledTimes(1);
15+
const [url, init] = mockFetch.mock.calls[0];
16+
return { url, init, body: await new Response(init?.body).json() };
17+
};
1418

1519
describe('sendUsageStats', () => {
20+
const originalEnv = { ...process.env };
21+
22+
beforeAll(() => {
23+
// Headers meant for the operator's collector, which must not reach
24+
// HyperDX.
25+
process.env.OTEL_EXPORTER_OTLP_HEADERS =
26+
'x-secret=1,Authorization=self-key';
27+
process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS = 'x-logs=2';
28+
});
29+
1630
beforeEach(() => {
17-
mockExporter.reset();
31+
mockFetch.mockReset();
32+
mockFetch.mockResolvedValue(new Response(null, { status: 200 }));
1833
});
1934

2035
afterEach(() => {
2136
jest.restoreAllMocks();
2237
});
2338

24-
it('exports one record to the usage stats endpoint with its own key', async () => {
39+
afterAll(() => {
40+
process.env = originalEnv;
41+
});
42+
43+
it('posts one record to the usage stats endpoint with only its own key', async () => {
44+
const timeoutSpy = jest.spyOn(AbortSignal, 'timeout');
2545
// A traced caller, so the record would pick up its span without an
2646
// explicit context.
2747
jest.spyOn(context, 'active').mockReturnValue(
@@ -35,47 +55,81 @@ describe('sendUsageStats', () => {
3555
await sendUsageStats({
3656
clusterId: 'cluster-1',
3757
userCounts: 2,
38-
chStats: { tables: { otel_logs: { rows: 10 } }, rows: 10 },
58+
chStats: { rows: 10 },
3959
});
4060

41-
expect(OTLPLogExporter).toHaveBeenCalledWith(
42-
expect.objectContaining({
43-
url: 'https://in-otel.hyperdx.io/v1/logs',
44-
headers: { authorization: expect.any(String) },
45-
}),
46-
);
61+
const { url, init, body } = await sentRequest();
62+
expect(url).toBe(USAGE_STATS_URL);
63+
expect(init?.method).toBe('POST');
64+
expect(init?.headers).toEqual({
65+
authorization: USAGE_STATS_API_KEY,
66+
'content-type': 'application/json',
67+
});
68+
expect(timeoutSpy).toHaveBeenCalledWith(10_000);
4769

48-
const records = mockExporter.getFinishedLogRecords();
49-
expect(records).toHaveLength(1);
50-
const [record] = records;
51-
expect(record.body).toBe('track-hyperdx-oss-usage-stats');
70+
expect(body.resourceLogs).toHaveLength(1);
71+
const [resourceLogs] = body.resourceLogs;
72+
expect(resourceLogs.resource.attributes).toContainEqual({
73+
key: 'service.name',
74+
value: { stringValue: 'hyperdx-oss-usage-stats' },
75+
});
76+
expect(resourceLogs.scopeLogs).toHaveLength(1);
77+
const [scopeLogs] = resourceLogs.scopeLogs;
78+
expect(scopeLogs.scope.name).toBe('node-logger');
79+
expect(scopeLogs.logRecords).toHaveLength(1);
80+
const [record] = scopeLogs.logRecords;
81+
expect(record.body).toEqual({
82+
stringValue: 'track-hyperdx-oss-usage-stats',
83+
});
5284
expect(record.severityNumber).toBe(SeverityNumber.INFO);
5385
expect(record.severityText).toBe('info');
54-
expect(record.instrumentationScope.name).toBe('node-logger');
55-
expect(record.spanContext).toBeUndefined();
56-
expect(record.attributes).toEqual({
57-
clusterId: 'cluster-1',
58-
userCounts: 2,
59-
chStats: { tables: { otel_logs: { rows: 10 } }, rows: 10 },
60-
time: expect.any(Number),
61-
pid: process.pid,
62-
hostname: os.hostname(),
63-
});
64-
expect(record.resource.attributes['service.name']).toBe(
65-
'hyperdx-oss-usage-stats',
66-
);
86+
expect(record).not.toHaveProperty('traceId');
87+
expect(record).not.toHaveProperty('spanId');
88+
expect(record.attributes).toEqual([
89+
{ key: 'clusterId', value: { stringValue: 'cluster-1' } },
90+
{ key: 'userCounts', value: { intValue: 2 } },
91+
{
92+
key: 'chStats',
93+
value: {
94+
kvlistValue: { values: [{ key: 'rows', value: { intValue: 10 } }] },
95+
},
96+
},
97+
{ key: 'time', value: { intValue: expect.any(Number) } },
98+
{ key: 'pid', value: { intValue: process.pid } },
99+
{ key: 'hostname', value: { stringValue: os.hostname() } },
100+
]);
67101
});
68102

69103
it('sends NaN as null and drops undefined fields, as the pino records did', async () => {
70104
await sendUsageStats({
71105
clusterId: undefined,
72-
chStats: { tables: { otel_logs: { avgDaySize: NaN, rows: 10 } } },
106+
chStats: { avgDaySize: NaN, rows: 10 },
73107
});
74108

75-
const [record] = mockExporter.getFinishedLogRecords();
76-
expect(record.attributes).not.toHaveProperty('clusterId');
77-
expect(record.attributes.chStats).toEqual({
78-
tables: { otel_logs: { avgDaySize: null, rows: 10 } },
109+
const { body } = await sentRequest();
110+
const [record] = body.resourceLogs[0].scopeLogs[0].logRecords;
111+
expect(record.attributes).not.toContainEqual(
112+
expect.objectContaining({ key: 'clusterId' }),
113+
);
114+
expect(record.attributes).toContainEqual({
115+
key: 'chStats',
116+
value: {
117+
kvlistValue: {
118+
values: [
119+
// An empty AnyValue, which is how OTLP JSON carries null
120+
{ key: 'avgDaySize', value: {} },
121+
{ key: 'rows', value: { intValue: 10 } },
122+
],
123+
},
124+
},
79125
});
80126
});
127+
128+
it('resolves when the request fails', async () => {
129+
mockFetch.mockRejectedValue(new Error('connect ECONNREFUSED'));
130+
await expect(sendUsageStats({ userCounts: 1 })).resolves.toBeUndefined();
131+
132+
mockFetch.mockResolvedValue(new Response(null, { status: 503 }));
133+
await expect(sendUsageStats({ userCounts: 1 })).resolves.toBeUndefined();
134+
});
81135
});

‎packages/api/src/tasks/index.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,24 @@ async function main(argv: TaskArgs): Promise<void> {
5555
});
5656
}
5757

58+
let cronJob: CronJob | undefined;
59+
let exiting = false;
60+
5861
// process.exit() skips the SDK's exit-time flush, so a one-shot task would
5962
// drop its own span and duration metric. Before init shutdown() resolves at
60-
// once.
63+
// once. The first call decides the exit code: a later exception or rejection
64+
// must not start a second shutdown or exit with another code.
6165
const exitAfterShutdown = (code: number) => {
66+
if (exiting) {
67+
return;
68+
}
69+
exiting = true;
70+
// Not awaited: with waitForCompletion it waits for the running tick, which
71+
// may be the one that failed.
72+
void cronJob?.stop();
73+
// The SDK's shutdown timer does not hold the event loop open, so the
74+
// process can end on its own before process.exit() runs.
75+
process.exitCode = code;
6276
void shutdown().finally(() => process.exit(code));
6377
};
6478

@@ -77,7 +91,7 @@ const instrumentedMain = timeExec(main, duration => {
7791
if (!RUN_SCHEDULED_TASKS_EXTERNALLY) {
7892
logger.info('In-app cron job is enabled');
7993
// run cron job every 1 minute
80-
CronJob.from({
94+
cronJob = CronJob.from({
8195
cronTime: '0 * * * * *',
8296
waitForCompletion: true,
8397
onTick: async () => instrumentedMain(argv),

0 commit comments

Comments
 (0)