Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions src/gateway/system-metrics.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,29 @@ describe('SystemMetricsService', () => {
cpuCount: 8,
});
});
it('starts host and disk reads concurrently before deriving free memory', async () => {
const { service, docker } = makeService();
let releaseHost!: (value: HostInfo) => void;
const hostInfo = new Promise<HostInfo>((resolve) => {
releaseHost = resolve;
});
docker.getHostInfo.mockReturnValue(hostInfo);
docker.getHostDiskInfo.mockResolvedValue({ totalDiskMb: 10000, freeDiskMb: 7000 });
docker.getHostFreeMemoryMb.mockReturnValue(1024);

const snapshot = service.collectSnapshot();
expect(docker.getHostInfo).toHaveBeenCalledTimes(1);
expect(docker.getHostDiskInfo).toHaveBeenCalledTimes(1);
expect(docker.getHostFreeMemoryMb).not.toHaveBeenCalled();

releaseHost({ totalRamMb: 4096, cpuCount: 8 });
await expect(snapshot).resolves.toEqual({
totalRamMb: 4096,
usedRamMb: 3072,
freeDiskMb: 7000,
cpuCount: 8,
});
});

it.each([
['host info null', () => ({ totalRamMb: null, cpuCount: 8 })],
Expand Down
6 changes: 4 additions & 2 deletions src/gateway/system-metrics.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@ export class SystemMetricsService {
let freeRamMb: number | null;

try {
hostInfo = await this.dockerService.getHostInfo();
diskInfo = await this.dockerService.getHostDiskInfo();
[hostInfo, diskInfo] = await Promise.all([
this.dockerService.getHostInfo(),
this.dockerService.getHostDiskInfo(),
]);
freeRamMb = this.dockerService.getHostFreeMemoryMb();
} catch {
return null;
Expand Down
65 changes: 57 additions & 8 deletions src/servers/servers.service.spec.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
type ServerMutation = Partial<Server>;

type ServerResourceRow = Pick<Server, 'id' | 'memoryLimitMb'>;
type ServerAllocationRow = { totalMemoryMb: number | string | null };

type ServerCountRow = { total: number };

type ServerQueryRow = Server | ServerResourceRow | ServerCountRow;
type ServerQueryRow = Server | ServerAllocationRow | ServerCountRow;

/** Minimal projection returned by listRequestableServers (requestable discovery). */
type RequestableRow = {
Expand All @@ -27,6 +27,7 @@ import {
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Test, type TestingModule } from '@nestjs/testing';
import { PgDialect } from 'drizzle-orm/pg-core';
import { DRIZZLE } from 'src/db/db.module';
import type { Server } from 'src/db/schema';
import {
Expand Down Expand Up @@ -461,7 +462,7 @@ describe('ServersService', () => {
});

it('returns the exact memory InsufficientResources payload after locked allocation', async () => {
selectResults.push([{ id: 'other', memoryLimitMb: 4000 }]);
selectResults.push([{ totalMemoryMb: 4000 }]);

let error: HttpException | undefined;
try {
Expand All @@ -486,15 +487,42 @@ describe('ServersService', () => {
expect(docker.createContainer).not.toHaveBeenCalled();
});

it('sums every existing server regardless of status for the create allocation', async () => {
selectResults.push([{ id: 'stopped', memoryLimitMb: 4000 }]);
it('uses a SQL allocation aggregate for all server statuses', async () => {
selectResults.push([{ totalMemoryMb: 4000 }]);
insertResults.push(makeServer({ status: 'CREATING', containerId: null }));
updateResults.push([makeServer({ status: 'CREATING' })], [makeServer({ status: 'RUNNING' })]);

await service.createServer(makeDto(), adminPrincipal);

// the create allocation query must not filter STOPPED rows
expect(queryChains[0].where).not.toHaveBeenCalled();
expect(db.select).toHaveBeenCalledWith({ totalMemoryMb: expect.anything() });
const aggregateSelection = db.select.mock.calls[0][0];
expect(new PgDialect().sqlToQuery(aggregateSelection.totalMemoryMb).sql).toBe(
'coalesce(sum("servers"."memory_limit_mb"), 0)',
);
});

it('treats the SQL aggregate zero-row result as zero allocation', async () => {
selectResults.push([]);
insertResults.push(makeServer({ status: 'CREATING', containerId: null }));
updateResults.push([makeServer({ status: 'CREATING' })], [makeServer({ status: 'RUNNING' })]);

await expect(service.createServer(makeDto(), adminPrincipal)).resolves.toEqual(
expect.objectContaining({ status: 'RUNNING' }),
);
});

it('filters the excluded server and stopped servers in start allocation queries', async () => {
selectResults.push([makeServer({ status: 'STOPPED' })], [{ totalMemoryMb: '4000' }]);
updateResults.push([makeServer({ status: 'STARTING' })], [makeServer({ status: 'RUNNING' })]);

await expect(service.startServer('server-1', adminPrincipal)).resolves.toEqual(
expect.objectContaining({ status: 'RUNNING' }),
);

const renderedWhere = new PgDialect().sqlToQuery(queryChains[1].where.mock.calls[0][0]);
expect(renderedWhere.sql).toBe('("servers"."status" <> $1 and "servers"."id" <> $2)');
expect(renderedWhere.params).toEqual(['STOPPED', 'server-1']);
});

it('fails closed for a null total RAM measurement', async () => {
Expand Down Expand Up @@ -552,6 +580,27 @@ describe('ServersService', () => {
);
expect(calls).toEqual(['hostInfo', 'diskInfo']);
});
it('starts host and disk preflight reads concurrently', async () => {
let releaseHost!: (value: HostInfo) => void;
const hostInfo = new Promise<HostInfo>((resolve) => {
releaseHost = resolve;
});
docker.getHostInfo = jest.fn(() => hostInfo);
docker.getHostDiskInfo = jest.fn().mockResolvedValue({
totalDiskMb: 100000,
freeDiskMb: 5000,
});

const request = service.createServer(makeDto(), adminPrincipal);
expect(docker.getHostInfo).toHaveBeenCalledTimes(1);
expect(docker.getHostDiskInfo).toHaveBeenCalledTimes(1);

releaseHost({ totalRamMb: null, cpuCount: 8 });
await expect(request).rejects.toMatchObject({
response: expect.objectContaining({ message: 'Host resource information unavailable' }),
status: 503,
});
});
});

describe('createServer', () => {
Expand Down Expand Up @@ -826,9 +875,9 @@ describe('ServersService', () => {
});

it('settles STOPPED when admission fails after a successful stop', async () => {
selectResults.push([makeServer({ status: 'RUNNING' })], []);
selectResults.push([makeServer({ status: 'RUNNING' })], [{ totalMemoryMb: 0 }]);
updateResults.push([makeServer({ status: 'STOPPING' })], [makeServer({ status: 'STOPPED' })]);
setResources(undefined, { totalDiskMb: 1000, freeDiskMb: 100 });
setResources({ totalRamMb: 8192, cpuCount: 8 }, { totalDiskMb: 1000, freeDiskMb: 100 });

await expect(service.restartServer('server-1', adminPrincipal)).rejects.toMatchObject({
status: 422,
Expand Down
42 changes: 23 additions & 19 deletions src/servers/servers.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,10 @@ export class ServersService implements OnModuleInit {

async createServer(dto: CreateServerDto, principal: ServerPrincipal): Promise<PublicServer> {
const config = this.parseResourceConfig();
const hostInfo = await this.dockerService.getHostInfo();
const diskInfo = await this.dockerService.getHostDiskInfo();

const [hostInfo, diskInfo] = await Promise.all([
this.dockerService.getHostInfo(),
this.dockerService.getHostDiskInfo(),
]);
this.checkDisk(diskInfo, config.minFreeDiskMb, 'Insufficient disk space to create server');

const created = await this.db.transaction(async (tx) => {
Expand Down Expand Up @@ -323,9 +324,10 @@ export class ServersService implements OnModuleInit {
}

const config = this.parseResourceConfig();
const hostInfo = await this.dockerService.getHostInfo();
const diskInfo = await this.dockerService.getHostDiskInfo();

const [hostInfo, diskInfo] = await Promise.all([
this.dockerService.getHostInfo(),
this.dockerService.getHostDiskInfo(),
]);
this.checkDisk(diskInfo, config.minFreeDiskMb, 'Insufficient disk space to start server');

const started = await this.db.transaction(async (tx) => {
Expand Down Expand Up @@ -456,9 +458,10 @@ export class ServersService implements OnModuleInit {

try {
const config = this.parseResourceConfig();
const hostInfo = await this.dockerService.getHostInfo();
const diskInfo = await this.dockerService.getHostDiskInfo();

const [hostInfo, diskInfo] = await Promise.all([
this.dockerService.getHostInfo(),
this.dockerService.getHostDiskInfo(),
]);
this.checkDisk(diskInfo, config.minFreeDiskMb, 'Insufficient disk space to restart server');

const starting = await this.db.transaction(async (tx) => {
Expand Down Expand Up @@ -742,17 +745,18 @@ export class ServersService implements OnModuleInit {
tx: Tx,
options: { excludeServerId?: string; includeStopped?: boolean } = { includeStopped: true },
): Promise<number> {
const rows = options.includeStopped
? await tx.select({ id: servers.id, memoryLimitMb: servers.memoryLimitMb }).from(servers)
: await tx
.select({ id: servers.id, memoryLimitMb: servers.memoryLimitMb })
.from(servers)
.where(ne(servers.status, 'STOPPED'));

return rows.reduce(
(sum, row) => (row.id !== options.excludeServerId ? sum + (row.memoryLimitMb ?? 0) : sum),
0,
const where = and(
options.includeStopped ? undefined : ne(servers.status, 'STOPPED'),
options.excludeServerId ? ne(servers.id, options.excludeServerId) : undefined,
);
const allocationQuery = tx
.select({
totalMemoryMb: sql<number>`coalesce(sum(${servers.memoryLimitMb}), 0)`,
})
.from(servers);
const [row] = await (where ? allocationQuery.where(where) : allocationQuery);

return Number(row?.totalMemoryMb ?? 0);
}

private parseResourceConfig(): ResourceConfig {
Expand Down