diff --git a/.changeset/pause-force.md b/.changeset/pause-force.md new file mode 100644 index 0000000000..2d3ad7d493 --- /dev/null +++ b/.changeset/pause-force.md @@ -0,0 +1,17 @@ +--- +'e2b': major +'@e2b/python-sdk': major +'@e2b/code-interpreter': major +'@e2b/desktop': major +'@e2b/code-interpreter-python': major +'@e2b/desktop-python': major +'@e2b/cli': minor +--- + +`pause()` now returns a result object instead of a boolean, and takes `force`. + +**Breaking:** `Sandbox.pause()` / `SandboxApi.pause()` (JavaScript) and `Sandbox.pause()` / `AsyncSandbox.pause()` (Python) return `SandboxPauseResult` with `paused` (`true` if this call paused the sandbox, `false` if it was already paused) and `forced` (whether the snapshot was taken without the guest agent health check, as reported by the API). Migrate by reading `paused` where the boolean was read: `if (await sandbox.pause())` becomes `if ((await sandbox.pause()).paused)` and `if sandbox.pause():` becomes `if sandbox.pause().paused:`. A plain truthiness check on the result is now always true, so update every call site that branched on the old boolean. `@e2b/code-interpreter` and `@e2b/desktop` re-export `e2b` and subclass its `Sandbox`, and `e2b-code-interpreter` / `e2b-desktop` subclass the Python `Sandbox`, so their `pause()` changes the same way and they take the same major; the Python packages now allow `e2b<4.0.0`. + +The node may refuse a pause (HTTP 503 with `error_code: "sandbox_pause_refused"`): the sandbox keeps running and the refusal surfaces as `SandboxPauseRefusedError` (JavaScript) / `SandboxPauseRefusedException` (Python), a subclass of `ServiceBusyError` / `ServiceBusyException` whose message carries the reason. When the reason is an unresponsive sandbox guest agent, the new `force` option (`{ force: true }` / `force=True`) takes the snapshot without the health check; its memory may then not resume, and a resume with `onResume: 'reboot'` / `on_resume="reboot"` recovers the disk state. Unlike other 503s, a refusal carrying this code is not replayed by the SDK's transport retries, in either SDK: it surfaces after a single request. A `204` from an older API is still a successful, unforced pause. + +`e2b sandbox pause` takes `-f, --force` and says when the pause was forced. `SANDBOX_PAUSE_REFUSED_ERROR_CODE` is exported from both packages. diff --git a/packages/cli/src/commands/sandbox/pause.ts b/packages/cli/src/commands/sandbox/pause.ts index e11a11a5c7..ac2c116005 100644 --- a/packages/cli/src/commands/sandbox/pause.ts +++ b/packages/cli/src/commands/sandbox/pause.ts @@ -3,21 +3,38 @@ import * as commander from 'commander' import { ensureAPIKey } from 'src/api' import { asBold } from 'src/utils/format' import * as e2b from 'e2b' -import { NotFoundError } from 'e2b' +import { NotFoundError, ServiceBusyError } from 'e2b' -async function pauseSandbox(sandboxID: string, apiKey: string) { +async function pauseSandbox( + sandboxID: string, + apiKey: string, + force: boolean | undefined +) { try { - const paused = await e2b.Sandbox.pause(sandboxID, { apiKey }) - if (paused) { - console.log(`Sandbox ${asBold(sandboxID)} has been paused`) - } else { + // Left out unless given, so the SDK omits it from the request body. + const { paused, forced } = await e2b.Sandbox.pause(sandboxID, { + apiKey, + ...(force ? { force: true } : {}), + }) + if (!paused) { console.log(`Sandbox ${asBold(sandboxID)} is already paused`) + } else if (forced) { + console.log( + `Sandbox ${asBold(sandboxID)} has been paused with force: the snapshot was taken without the guest agent health check, so its memory may not resume (a reboot resume recovers the disk state)` + ) + } else { + console.log(`Sandbox ${asBold(sandboxID)} has been paused`) } } catch (err: unknown) { if (err instanceof NotFoundError) { console.error(`Sandbox ${asBold(sandboxID)} wasn't found`) process.exit(1) } + if (err instanceof ServiceBusyError) { + // The API's message says what to do: retry, or pause with --force. + console.error(err.message) + process.exit(1) + } throw err } } @@ -29,10 +46,14 @@ export const pauseCommand = new commander.Command('pause') `pause the sandbox specified by ${asBold('')}` ) .alias('ps') - .action(async (sandboxID: string) => { + .option( + '-f, --force', + 'pause even when the sandbox guest agent is unresponsive: the snapshot is taken without the health check and its memory may not resume (a reboot resume recovers the disk state)' + ) + .action(async (sandboxID: string, { force }: { force?: boolean }) => { try { const apiKey = ensureAPIKey() - await pauseSandbox(sandboxID, apiKey) + await pauseSandbox(sandboxID, apiKey, force) } catch (err: unknown) { console.error(err) process.exit(1) diff --git a/packages/cli/tests/commands/sandbox/pause.test.ts b/packages/cli/tests/commands/sandbox/pause.test.ts new file mode 100644 index 0000000000..8855b7f324 --- /dev/null +++ b/packages/cli/tests/commands/sandbox/pause.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + pause: vi.fn(), + ensureAPIKey: vi.fn(() => 'test-api-key'), +})) + +class ServiceBusyError extends Error {} +class SandboxPauseRefusedError extends ServiceBusyError {} + +vi.mock('e2b', () => ({ + Sandbox: { + pause: mocks.pause, + }, + NotFoundError: class NotFoundError extends Error {}, + ServiceBusyError, +})) + +vi.mock('../../../src/api', () => ({ + ensureAPIKey: mocks.ensureAPIKey, +})) + +describe('sandbox pause', () => { + let logSpy: ReturnType + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + async function run(...args: string[]) { + const { pauseCommand } = await import('../../../src/commands/sandbox/pause') + await pauseCommand.parseAsync(['node', 'e2b', ...args]) + } + + test('pauses without force by default, leaving force out of the options', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: false }) + + await run('sbx-1') + + expect(mocks.pause).toHaveBeenCalledWith('sbx-1', { + apiKey: 'test-api-key', + }) + expect(mocks.pause.mock.calls[0][1]).not.toHaveProperty('force') + expect(logSpy.mock.calls.flat().join('\n')).toContain('has been paused') + expect(logSpy.mock.calls.flat().join('\n')).not.toContain('with force') + }) + + test('--force passes force and reports a forced pause', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: true }) + + await run('sbx-1', '--force') + + expect(mocks.pause).toHaveBeenCalledWith('sbx-1', { + apiKey: 'test-api-key', + force: true, + }) + expect(logSpy.mock.calls.flat().join('\n')).toContain( + 'has been paused with force' + ) + }) + + test('a forced request the API did not force is reported as a plain pause', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: false }) + + await run('sbx-1', '-f') + + expect(logSpy.mock.calls.flat().join('\n')).not.toContain('with force') + }) + + test('a refused pause prints the API message, not a stack, and exits 1', async () => { + const message = + "Sandbox 'sbx-1' cannot be paused right now because the sandbox guest agent is not responding: retry, or pause with force" + mocks.pause.mockRejectedValue(new SandboxPauseRefusedError(message)) + const errorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => undefined) + const exitSpy = vi + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as never) + + await run('sbx-1') + + // The first thing printed is the API message itself; with process.exit + // mocked the command carries on past it, so only the first call counts. + expect(errorSpy.mock.calls[0]).toEqual([message]) + expect(exitSpy).toHaveBeenNthCalledWith(1, 1) + }) + + test('an already paused sandbox is reported as such', async () => { + mocks.pause.mockResolvedValue({ paused: false, forced: false }) + + await run('sbx-1') + + expect(logSpy.mock.calls.flat().join('\n')).toContain('is already paused') + }) +}) diff --git a/packages/code-interpreter-python/pyproject.toml b/packages/code-interpreter-python/pyproject.toml index c86020a5df..cf4773d7d7 100644 --- a/packages/code-interpreter-python/pyproject.toml +++ b/packages/code-interpreter-python/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.11" dependencies = [ "httpx>=0.20.0,<1.0.0", "attrs>=21.3.0", - "e2b>=2.55.1,<3.0.0", + "e2b>=2.55.1,<4.0.0", ] [project.urls] diff --git a/packages/desktop-python/pyproject.toml b/packages/desktop-python/pyproject.toml index 6c91abd6d4..595408ac72 100644 --- a/packages/desktop-python/pyproject.toml +++ b/packages/desktop-python/pyproject.toml @@ -7,7 +7,7 @@ license = "MIT" readme = "README.md" requires-python = ">=3.11" dependencies = [ - "e2b>=2.44.0,<3.0.0", + "e2b>=2.44.0,<4.0.0", "requests>=2.32.3,<3", "urllib3>=2.8.0,<3", "pillow>=12.0.0,<13", diff --git a/packages/js-sdk/src/api/index.ts b/packages/js-sdk/src/api/index.ts index 23d9a06307..c4118c7930 100644 --- a/packages/js-sdk/src/api/index.ts +++ b/packages/js-sdk/src/api/index.ts @@ -7,6 +7,8 @@ import { ConnectionConfig } from '../connectionConfig' import { AuthenticationError, RateLimitError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, + SandboxPauseRefusedError, ServiceBusyError, SandboxError, } from '../errors' @@ -16,7 +18,9 @@ import { withRetry } from '../retry' /** * Map an API error code and message to the matching error class — the same * mapping {@link handleApiError} applies to HTTP responses, usable for error - * objects embedded in response bodies (e.g. per-fork results). + * objects embedded in response bodies (e.g. per-fork results). `errorCode` is + * the body's semantic `error_code`, which refines the status for the codes + * the SDK types (a refused pause). */ export function apiErrorFromCode( code: number, @@ -25,7 +29,8 @@ export function apiErrorFromCode( message: string, stackTrace?: string ) => Error = SandboxError, - stackTrace?: string + stackTrace?: string, + errorCode?: string ): Error { if (code === 401) { const message = 'Unauthorized, please check your credentials.' @@ -40,6 +45,12 @@ export function apiErrorFromCode( } if (code === 503) { + if (errorCode === SANDBOX_PAUSE_REFUSED_ERROR_CODE) { + // The API's message already says what to do for its reason. + return new SandboxPauseRefusedError( + content ? `${content}` : 'The node refused the pause, please retry' + ) + } const message = 'Service temporarily unavailable, please retry' return new ServiceBusyError(content ? `${message} - ${content}` : message) } @@ -72,7 +83,8 @@ export function handleApiError( status, response.error?.message ?? response.error, errorClass, - stackTrace + stackTrace, + response.error?.error_code ) } diff --git a/packages/js-sdk/src/api/schema.gen.ts b/packages/js-sdk/src/api/schema.gen.ts index cb39bcc138..952bc6ef3e 100644 --- a/packages/js-sdk/src/api/schema.gen.ts +++ b/packages/js-sdk/src/api/schema.gen.ts @@ -449,12 +449,15 @@ export interface paths { }; responses: { /** @description The sandbox was paused successfully and can be resumed */ - 204: { + 200: { headers: { [name: string]: unknown; }; - content?: never; + content: { + "application/json": components["schemas"]["SandboxPauseResult"]; + }; }; + 400: components["responses"]["400"]; 401: components["responses"]["401"]; 404: components["responses"]["404"]; 409: components["responses"]["409"]; @@ -2258,7 +2261,7 @@ export interface components { * @description Error code */ code: number; - /** @description Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, internal_server_error, secret_limit_reached. */ + /** @description Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, sandbox_pause_refused (the node refused the pause and the sandbox keeps running: the guest agent did not answer the health check, or the node was busy; retry, or pause with force when the guest agent is unresponsive), internal_server_error, secret_limit_reached. */ error_code?: string; /** @description Error */ message: string; @@ -2711,12 +2714,21 @@ export interface components { */ SandboxOnTimeout: "kill" | "pause"; SandboxPauseRequest: { + /** + * @description Take the pause even when the sandbox guest agent is unresponsive. Applies to memory pauses only. The memory snapshot may not resume; a reboot resume recovers the disk state. Defaults to false. + * @default false + */ + force?: boolean; /** * @description Whether to capture a full memory snapshot. When false, only the filesystem is persisted and resuming the sandbox cold-boots (reboots) it from disk, losing in-memory state, running processes, and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. * @default true */ memory?: boolean; }; + SandboxPauseResult: { + /** @description Whether the pause was taken with the force override, skipping the guest agent health check. */ + forced: boolean; + }; SandboxRefreshRequest: { /** @description Duration for which the sandbox should be kept alive in seconds */ duration?: number; diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 176fed680f..5383a07e2a 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -261,6 +261,41 @@ export class ServiceBusyError extends Error { } } +/** + * `error_code` of the 503 the API answers a pause with when the node refused + * it. Any endpoint answering this code is mapped to + * {@link SandboxPauseRefusedError}. + */ +export const SANDBOX_PAUSE_REFUSED_ERROR_CODE = 'sandbox_pause_refused' + +/** + * Thrown when the node refused a pause (HTTP 503 with + * `error_code: "sandbox_pause_refused"`). The sandbox keeps running with its + * state intact; the message is the API's, and names the reason and what to + * do about it. In general: retry, and when the reason is an unresponsive + * sandbox guest agent, pause with `force: true` to take the snapshot without + * the health check — its memory may then not resume, and a resume with + * `onResume: 'reboot'` recovers the disk state. + * + * Unlike other 503s, a refusal carrying this code is not replayed by the + * SDK's transport retries: it is a decision, not transient capacity, so it + * surfaces after a single request and the caller decides what to do. + * + * Subclass of {@link ServiceBusyError}, which a refused pause surfaced as + * before, so existing handlers keep working. + */ +export class SandboxPauseRefusedError extends ServiceBusyError { + /** + * Semantic error code of the API response: always `sandbox_pause_refused`. + */ + readonly errorCode = SANDBOX_PAUSE_REFUSED_ERROR_CODE + + constructor(message: string) { + super(message) + this.name = 'SandboxPauseRefusedError' + } +} + /** * Thrown when the build fails. */ diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index 55a74e6af3..07ada50f4f 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -24,6 +24,8 @@ export { TimeoutError, RateLimitError, ServiceBusyError, + SandboxPauseRefusedError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, BuildError, FileUploadError, VolumeError, @@ -68,6 +70,7 @@ export type { SandboxForkOpts, SandboxMetricsOpts, SandboxPauseOpts, + SandboxPauseResult, SandboxState, SandboxListOpts, SandboxListOrder, diff --git a/packages/js-sdk/src/retry.ts b/packages/js-sdk/src/retry.ts index 903edce8bb..b6961de807 100644 --- a/packages/js-sdk/src/retry.ts +++ b/packages/js-sdk/src/retry.ts @@ -1,4 +1,7 @@ -import { InvalidArgumentError } from './errors' +import { + InvalidArgumentError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, +} from './errors' import { isReadableStreamLike } from './is' const MAX_RETRY_AFTER_SECONDS = 2_147_483 @@ -7,6 +10,9 @@ const BACKOFF_BASE_MS = 100 const BACKOFF_MAX_MS = 10_000 const BACKOFF_JITTER_MIN = 0.5 const RETRYABLE_STATUSES = new Set([429, 502, 503]) +// A 503 whose body carries one of these `error_code`s is a decision by the +// server, not transient capacity, and is answered after a single request. +const NON_RETRYABLE_ERROR_CODES = new Set([SANDBOX_PAUSE_REFUSED_ERROR_CODE]) // Error codes raised while establishing the connection, before any request // bytes are written: Node/undici (`ECONNREFUSED`, `ENOTFOUND`, ...), Bun // (`ConnectionRefused`). Post-write failures such as `ECONNRESET` are @@ -160,6 +166,113 @@ function backoffMs(attempt: number, random: () => number): number { ) } +const BODY_READ_TIMED_OUT = Symbol('body read timed out') + +/** The media type of a `Content-Type` value: lower-cased, parameters dropped. */ +function mediaType(contentType: string | null): string { + return (contentType ?? '').split(';', 1)[0].trim().toLowerCase() +} +// An error body larger than this cannot carry a code worth parsing; the read +// stops there and the 503 is retried as a plain one. +const MAX_ERROR_BODY_BYTES = 8 * 1024 +// Time a 503 body is peeked at for a non-retryable code, independent of the +// jittered backoff: a slow refusal body within this still short-circuits, and +// a trickling non-refusal body costs at most this per attempt. +const PEEK_BUDGET_MS = 1_000 + +/** + * Reads `reader` to the end into one buffer, or returns `undefined` once more + * than `limit` bytes arrived. + */ +async function readUpTo( + reader: ReadableStreamDefaultReader, + limit: number +): Promise { + const chunks: Uint8Array[] = [] + let size = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + size += value.byteLength + if (size > limit) return undefined + chunks.push(value) + } + const out = new Uint8Array(size) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.byteLength + } + return out +} + +/** + * Whether a 503 carries a non-retryable `error_code` (see + * `NON_RETRYABLE_ERROR_CODES`). Reads a clone through a reader this function + * owns, capped at `MAX_ERROR_BODY_BYTES` and bounded by `budgetMs` (the + * inspection budget, which overlaps the retry wait instead of adding to it; + * waited through `sleep` so the injected clock governs it) and the request + * signal; + * when it stops early the reader is cancelled so the abandoned read does not + * keep consuming the stream, and the answer is "not a refusal" so the 503 is + * retried as before. A body that cannot be peeked at (none, not JSON, + * consumed, locked) or a request already aborted is not a refusal either. The original body + * stays untouched for the caller. + */ +async function isNonRetryableRefusal( + response: Response, + budgetMs: number, + signal: AbortSignal, + sleep: (delayMs: number, signal: AbortSignal) => Promise +): Promise { + if (response.status !== 503 || signal.aborted) return false + // Only a JSON body can carry the code; a proxy's HTML page is not peeked at. + if (mediaType(response.headers.get('Content-Type')) !== 'application/json') { + return false + } + const { body } = response + if (!body || response.bodyUsed || body.locked) return false + let reader: ReadableStreamDefaultReader | undefined + const timer = new AbortController() + const onAbort = () => timer.abort(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + // Clamped so a huge budget cannot overflow setTimeout into a 1 ms timer. + const bounded = Math.min( + Math.max(0, budgetMs), + MAX_RETRY_AFTER_SECONDS * 1000 + ) + try { + const cloned = response.clone().body + if (!cloned) return false + reader = cloned.getReader() + const bytes = await Promise.race([ + readUpTo(reader, MAX_ERROR_BODY_BYTES).catch(() => undefined), + sleep(bounded, timer.signal).then( + () => BODY_READ_TIMED_OUT, + () => BODY_READ_TIMED_OUT + ), + ]) + if (typeof bytes === 'symbol' || bytes === undefined) { + // Settles the pending read with done, so readUpTo ends too. + void reader.cancel().catch(() => {}) + return false + } + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as { + error_code?: unknown + } + return ( + typeof parsed?.error_code === 'string' && + NON_RETRYABLE_ERROR_CODES.has(parsed.error_code) + ) + } catch { + void reader?.cancel().catch(() => {}) + return false + } finally { + timer.abort() + signal.removeEventListener('abort', onAbort) + } +} + /** * Delay before the next attempt, or `undefined` when the response is not * retried: `Retry-After` when the server sends a usable one, otherwise @@ -187,7 +300,10 @@ function retryDelayMs( * Retry requests after a 429 carrying `Retry-After`, a 503 or — for replayable * requests ({@link isReplayable}) — a 502 (using `Retry-After` when present, * exponential backoff otherwise), or a network failure (exponential backoff; - * see {@link isRetryableFetchError}). + * see {@link isRetryableFetchError}). A 503 whose body carries a + * non-retryable `error_code` (a refused pause) is returned after one request; + * the body is peeked at for an inspection budget of max(retry delay, 1 s) + * within the remaining request budget. */ export function withRetry( fetchImpl: typeof fetch, @@ -246,10 +362,27 @@ export function withRetry( return response } + // Peek at a 503 body for a non-retryable code for its own inspection + // budget (at least the retry delay, within the remaining request + // budget); the time spent peeking counts towards the delay. + const peekStart = monotonic() + if ( + await isNonRetryableRefusal( + response, + Math.min(Math.max(delayMs, PEEK_BUDGET_MS), deadline - peekStart), + request.signal, + sleep + ) + ) { + return response + } + const remainingDelayMs = Math.max(0, delayMs - (monotonic() - peekStart)) + if (monotonic() + remainingDelayMs >= deadline) return response + // Not awaited: some fetch interceptors never settle `cancel()`, and the // retry does not depend on the discarded body having been released. void response.body?.cancel().catch(() => {}) - await sleep(delayMs, request.signal) + await sleep(remainingDelayMs, request.signal) } }) as typeof fetch } diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts index 3af8e5e3d4..2f2fe2bccb 100644 --- a/packages/js-sdk/src/sandbox/index.ts +++ b/packages/js-sdk/src/sandbox/index.ts @@ -28,6 +28,7 @@ import { SnapshotPaginator, CreateSnapshotOpts, SandboxPauseOpts, + SandboxPauseResult, } from './sandboxApi' import { getSignature } from './signature' import { compareVersions } from 'compare-versions' @@ -639,31 +640,47 @@ export class Sandbox extends SandboxApi { /** * Pause a sandbox by its ID. * - * @param opts connection options, plus `mode` to control the snapshot kind. + * The node may refuse the pause ({@link SandboxPauseRefusedError}): the + * sandbox keeps running and the message carries the reason. When the reason + * is an unresponsive sandbox guest agent, `force: true` takes the snapshot + * without the health check, in which case the memory snapshot may not + * resume and a resume with `onResume: 'reboot'` recovers the disk state. + * + * @param opts connection options, plus `mode` to control the snapshot kind + * and `force` to skip the guest agent health check. * With `mode: 'filesystem'`, the in-memory state is dropped and only the * filesystem is persisted; resuming such a sandbox cold-boots (reboots) it * from disk, losing running processes and open connections. Defaults to * `'full'` (memory and filesystem). * - * @returns `true` if the sandbox got paused, `false` if the sandbox was already paused. + * @returns the {@link SandboxPauseResult}: `paused` is `true` if this call + * paused the sandbox and `false` if it was already paused; `forced` whether + * the snapshot was taken without the guest agent health check (reported by + * the API, so a forced request that joined a normal pause already in + * flight comes back with `forced: false`). + * + * @throws {@link SandboxPauseRefusedError} if the node refused the pause; the message carries the reason. * * @example * ```ts * const sandbox = await Sandbox.create() - * await sandbox.pause() + * const { paused } = await sandbox.pause() * * // filesystem-only snapshot (resume reboots the sandbox) * await sandbox.pause({ mode: 'filesystem' }) + * + * // snapshot an unresponsive sandbox anyway; its memory may not resume + * const { forced } = await sandbox.pause({ force: true }) * ``` */ - async pause(opts?: SandboxPauseOpts): Promise { + async pause(opts?: SandboxPauseOpts): Promise { return await SandboxApi.pause(this.sandboxId, this.resolveApiOpts(opts)) } /** * @deprecated Use {@link Sandbox.pause} instead. */ - async betaPause(opts?: SandboxPauseOpts): Promise { + async betaPause(opts?: SandboxPauseOpts): Promise { return await SandboxApi.betaPause(this.sandboxId, this.resolveApiOpts(opts)) } diff --git a/packages/js-sdk/src/sandbox/sandboxApi.ts b/packages/js-sdk/src/sandbox/sandboxApi.ts index 50872b4881..2f6b44170f 100644 --- a/packages/js-sdk/src/sandbox/sandboxApi.ts +++ b/packages/js-sdk/src/sandbox/sandboxApi.ts @@ -585,6 +585,42 @@ export interface SandboxPauseOpts extends SandboxApiOpts { * `mode: 'filesystem'`, `keepMemory: true` is `mode: 'full'`. */ keepMemory?: boolean + + /** + * Take the pause even when the sandbox guest agent is unresponsive. + * + * A memory pause first checks that the guest agent answers. When it does + * not, the node refuses the pause ({@link SandboxPauseRefusedError}, whose + * message carries the reason) and the sandbox keeps running. With + * `force: true` the snapshot is taken without that check. Its memory may + * then not resume: if restoring it fails, resume with `onResume: 'reboot'` + * to recover the disk state. A refusal for another reason is not lifted + * by `force`. + * + * Applies to memory pauses only; the API rejects it together with + * `mode: 'filesystem'`. + * + * @default false + */ + force?: boolean +} + +/** + * Result of a pause. + */ +export interface SandboxPauseResult { + /** + * `true` if this call paused the sandbox, `false` if it was already paused. + */ + paused: boolean + + /** + * Whether the snapshot was taken with the `force` override, without the + * guest agent health check. Reports what happened rather than echoing the + * option: `false` when the sandbox was already paused, and also when a + * forced request joined a normal pause that was already in flight. + */ + forced: boolean } /** @@ -1588,15 +1624,25 @@ export class SandboxApi extends ClientFactory { /** * Pause the sandbox specified by sandbox ID. * + * The node may refuse the pause ({@link SandboxPauseRefusedError}): the + * sandbox keeps running and the message carries the reason. When the reason + * is an unresponsive sandbox guest agent, `force: true` takes the snapshot + * without the health check, in which case the memory snapshot may not + * resume and a resume with `onResume: 'reboot'` recovers the disk state. + * * @param sandboxId sandbox ID. - * @param opts pause options, including `mode` and connection options. + * @param opts pause options, including `mode`, `force` and connection options. * - * @returns `true` if the sandbox got paused, `false` if the sandbox was already paused. + * @returns the {@link SandboxPauseResult}: `paused` is `true` if this call + * paused the sandbox and `false` if it was already paused; `forced` whether + * the snapshot was taken without the guest agent health check. + * + * @throws {@link SandboxPauseRefusedError} if the node refused the pause; the message carries the reason. */ static async pause( sandboxId: string, opts?: SandboxPauseOpts - ): Promise { + ): Promise { const apiOpts = this.resolveOpts(opts) const config = new ConnectionConfig(apiOpts) const client = new ApiClient(config) @@ -1609,6 +1655,7 @@ export class SandboxApi extends ClientFactory { }, body: { memory: snapshotMemory('mode', apiOpts?.mode, apiOpts?.keepMemory), + force: apiOpts?.force, }, signal: config.getSignal(apiOpts?.requestTimeoutMs, apiOpts?.signal), }) @@ -1619,7 +1666,7 @@ export class SandboxApi extends ClientFactory { if (res.error?.code === 409) { // Sandbox is already paused - return false + return { paused: false, forced: false } } const err = handleApiError(res) @@ -1627,7 +1674,8 @@ export class SandboxApi extends ClientFactory { throw err } - return true + // An older API answers 204 without a body. + return { paused: true, forced: res.data?.forced ?? false } } /** @@ -1636,7 +1684,7 @@ export class SandboxApi extends ClientFactory { static async betaPause( sandboxId: string, opts?: SandboxPauseOpts - ): Promise { + ): Promise { return this.pause(sandboxId, opts) } @@ -1912,7 +1960,13 @@ export class SandboxApi extends ClientFactory { `${result.error.code}: ${result.error.message}` ) } - return apiErrorFromCode(result.error.code, result.error.message) + return apiErrorFromCode( + result.error.code, + result.error.message, + undefined, + undefined, + result.error.error_code + ) } return toSandboxCreateResponse(result.sandbox) diff --git a/packages/js-sdk/tests/api/handleApiError.test.ts b/packages/js-sdk/tests/api/handleApiError.test.ts index 31ad79ba41..ef4996e6bd 100644 --- a/packages/js-sdk/tests/api/handleApiError.test.ts +++ b/packages/js-sdk/tests/api/handleApiError.test.ts @@ -3,6 +3,7 @@ import { handleApiError } from '../../src/api' import { AuthenticationError, RateLimitError, + SandboxPauseRefusedError, ServiceBusyError, SandboxError, } from '../../src/errors' @@ -67,6 +68,35 @@ describe('handleApiError', () => { assert.include(err.message, 'node is busy persisting sandbox') }) + test('a 503 carrying error_code sandbox_pause_refused is a SandboxPauseRefusedError with the API message, whatever the endpoint', () => { + const res = createMockResponse(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message: + "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry", + }) + const err = handleApiError(res as any) as SandboxPauseRefusedError + assert.instanceOf(err, SandboxPauseRefusedError) + assert.instanceOf(err, ServiceBusyError) + assert.equal(err.statusCode, 503) + assert.equal(err.errorCode, 'sandbox_pause_refused') + assert.equal( + err.message, + "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry" + ) + }) + + test('a 503 carrying another error_code stays a ServiceBusyError', () => { + const res = createMockResponse(503, { + code: 503, + error_code: 'sandbox_capacity_unavailable', + message: 'no capacity', + }) + const err = handleApiError(res as any) + assert.instanceOf(err, ServiceBusyError) + assert.notInstanceOf(err, SandboxPauseRefusedError) + }) + test('a generic failure keeps SandboxError and carries its status', () => { const res = createMockResponse(500, { message: 'boom' }) const err = handleApiError(res as any) as SandboxError diff --git a/packages/js-sdk/tests/retry.test.ts b/packages/js-sdk/tests/retry.test.ts index 7f43e9f9e7..ac808abc20 100644 --- a/packages/js-sdk/tests/retry.test.ts +++ b/packages/js-sdk/tests/retry.test.ts @@ -237,6 +237,346 @@ test.each([ expect(fetchImpl).toHaveBeenCalledTimes(calls) }) +function refusal(errorCode: string): Response { + return new Response( + JSON.stringify({ code: 503, error_code: errorCode, message: 'refused' }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) +} + +// The peek at a 503 body waits through the injected sleep too, so a sleep that +// resolves at once ends every peek before an in-memory body can be read; the +// peek tests use a real, short timer instead. +const realSleep = (delayMs: number) => + new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, 100))) + +test('returns a 503 carrying a non-retryable error_code after one request, body intact', async () => { + const fetchImpl = vi.fn(async () => + refusal('sandbox_pause_refused') + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('a 503 whose body was already consumed is not peeked at and is retried', async () => { + const consumed = refusal('sandbox_pause_refused') + await consumed.text() + const responses = [consumed, new Response('ok')] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + +test('a 503 whose body is locked is not peeked at and is retried', async () => { + const locked = refusal('sandbox_pause_refused') + const reader = locked.body!.getReader() + const responses = [locked, new Response('ok')] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + reader.releaseLock() +}) + +test('a 503 is not peeked at when the request was already aborted', async () => { + const controller = new AbortController() + const responses = [refusal('sandbox_pause_refused'), new Response('ok')] + const fetchImpl = vi.fn(async () => { + controller.abort() + return responses.shift()! + }) as typeof fetch + // A sleep that ignores the signal, so the loop is observed past the peek. + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + }) + + await fetchWithRetry('https://api.e2b.test/sandboxes/sbx-1/pause', { + method: 'POST', + signal: controller.signal, + }) + + // Not short-circuited although the body carried the code, and not peeked + // at: the only sleep is the retry delay (a peek would have added one). + expect(fetchImpl).toHaveBeenCalledTimes(2) + expect(sleep).toHaveBeenCalledTimes(1) +}) + +test('a 503 whose body stalls is retried after its inspection budget (max(delay, 1 s)), not held for the whole request budget', async () => { + const fetchImpl = vi.fn( + async () => + new Response(new ReadableStream({ start() {} }), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }) + ) as typeof fetch + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + random: () => 0, + }) + + const startedAt = performance.now() + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + // Every attempt peeks for the inspection budget (1 s, above the 50, 100, + // 200 ms backoffs) through the injected sleep, then sleeps its backoff + // (the stub clock never advances, so nothing is subtracted), and retries. + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(4) + expect(sleep.mock.calls.map(([delayMs]) => delayMs)).toEqual([ + 1000, 50, 1000, 100, 1000, 200, + ]) + expect(performance.now() - startedAt).toBeLessThan(1_000) +}) + +test('a refusal that arrives within the delay still short-circuits', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + '{"code":503,"error_code":"sandbox_pause_refused","message":"refused"}' + ) + ) + controller.close() + }, 20) + }, + }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + random: () => 1, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + // Peeked at and returned: the caller's body is untouched and still readable. + expect(response.bodyUsed).toBe(false) + expect(response.body?.locked).toBe(false) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('a refusal body arriving after the first backoff but within the inspection budget still short-circuits', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + '{"code":503,"error_code":"sandbox_pause_refused","message":"refused"}' + ) + ) + controller.close() + }, 120) + }, + }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: (delayMs) => + new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, 500))), + // First backoff 50 ms: shorter than the body's 120 ms. + random: () => 0, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('the content type is matched by media type, whatever its case and parameters', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + JSON.stringify({ code: 503, error_code: 'sandbox_pause_refused' }), + { + status: 503, + headers: { 'Content-Type': 'Application/JSON; charset=UTF-8' }, + } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) +}) + +test('a 503 whose body keeps dripping past the inspection budget is retried and the clone read is cancelled', async () => { + let served = 0 + let cancelled = 0 + const drip = () => { + let stopped = false + let pending: ReturnType | undefined + return new ReadableStream({ + pull(controller) { + return new Promise((resolve) => { + pending = setTimeout(() => { + // The source is only told about one cancel; after it the + // controller is closed and must not be enqueued into. + if (!stopped) { + served++ + controller.enqueue(new TextEncoder().encode('{"error_code":"x')) + } + resolve() + }, 5) + }) + }, + cancel() { + stopped = true + cancelled++ + clearTimeout(pending) + }, + }) + } + const fetchImpl = vi.fn( + async () => + new Response(drip(), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }) + ) as typeof fetch + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + random: () => 0, + }) + + const startedAt = performance.now() + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + const servedAtReturn = served + + // The last response is returned without a peek (no retry left), so the + // untouched-body assertions live in the refusal-within-the-delay test. + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(4) + expect(sleep).toHaveBeenCalledTimes(6) + expect(performance.now() - startedAt).toBeLessThan(1_000) + + // The three retried responses had both tee branches cancelled (the peek's + // reader and the discarded body), so their sources were cancelled; the + // returned one keeps the caller's branch alive. With the peek's reader + // cancelled nothing keeps pulling it: at most the tee's one-chunk + // high-water mark can still be served for the caller's branch. + await new Promise((resolve) => setTimeout(resolve, 150)) + expect(cancelled).toBe(3) + expect(served - servedAtReturn).toBeLessThanOrEqual(2) +}) + +test('a 503 whose body exceeds the size cap is a plain 503 and is retried', async () => { + const large = JSON.stringify({ + code: 503, + error_code: 'sandbox_pause_refused', + message: 'x'.repeat(9 * 1024), + }) + const responses = [ + new Response(large, { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }), + new Response('ok'), + ] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + +test('retries a 503 carrying another error_code', async () => { + const responses = [ + refusal('sandbox_capacity_unavailable'), + new Response('ok'), + ] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry('https://api.e2b.test/sandboxes', { + method: 'POST', + }) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + test('retries a 503 for a non-replayable POST', async () => { const statuses = [503, 200] const fetchImpl = vi.fn( diff --git a/packages/js-sdk/tests/sandbox/apiDefaults.test.ts b/packages/js-sdk/tests/sandbox/apiDefaults.test.ts index 3f98484460..9c7b457139 100644 --- a/packages/js-sdk/tests/sandbox/apiDefaults.test.ts +++ b/packages/js-sdk/tests/sandbox/apiDefaults.test.ts @@ -42,7 +42,7 @@ const server = setupMockApi( }), http.post(apiUrl('/sandboxes/:sandboxID/pause'), async ({ request }) => { lastPauseBody = (await request.json()) as Record - return new HttpResponse(null, { status: 204 }) + return HttpResponse.json({ forced: lastPauseBody.force === true }) }), http.post(apiUrl('/sandboxes/:sandboxID/snapshots'), async ({ request }) => { lastSnapshotBody = (await request.json()) as Record @@ -140,6 +140,33 @@ test('Sandbox.pause omits memory when mode is unset', async () => { expect(lastPauseBody).not.toHaveProperty('memory') }) +test('Sandbox.pause omits force when unset', async () => { + await Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + + expect(lastPauseBody).toBeDefined() + expect(lastPauseBody).not.toHaveProperty('force') +}) + +test.each([true, false])( + 'Sandbox.pause sends an explicit force %s', + async (force) => { + await Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force }) + + expect(lastPauseBody?.force).toBe(force) + } +) + +test('Sandbox.pause sends force alongside mode', async () => { + await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + mode: 'full', + force: true, + }) + + expect(lastPauseBody?.memory).toBe(true) + expect(lastPauseBody?.force).toBe(true) +}) + test.each([ ['filesystem', false], ['full', true], diff --git a/packages/js-sdk/tests/sandbox/pause.test.ts b/packages/js-sdk/tests/sandbox/pause.test.ts new file mode 100644 index 0000000000..bec4ee663f --- /dev/null +++ b/packages/js-sdk/tests/sandbox/pause.test.ts @@ -0,0 +1,180 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'vitest' +import { http, HttpResponse } from 'msw' + +import { + Sandbox, + SandboxError, + SandboxNotFoundError, + SandboxPauseRefusedError, + ServiceBusyError, +} from '../../src' +import { TEST_API_KEY, apiUrl } from '../setup' +import { setupMockApi } from '../mockApi' + +const server = setupMockApi() + +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) + +afterAll(() => server.close()) + +afterEach(() => { + requests = 0 + server.resetHandlers() +}) + +let requests = 0 + +function pauseAnswers(status: number, body?: unknown) { + server.use( + http.post(apiUrl('/sandboxes/:sandboxID/pause'), () => { + requests++ + return body === undefined + ? new HttpResponse(null, { status }) + : HttpResponse.json(body, { status }) + }) + ) +} + +test('a 200 with forced false resolves to paused, not forced', async () => { + pauseAnswers(200, { forced: false }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('a 200 with forced true resolves to paused and forced', async () => { + pauseAnswers(200, { forced: true }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force: true }) + ).resolves.toEqual({ paused: true, forced: true }) +}) + +test('forced reports what the API did, not the option: a forced request that joined a normal pause is not forced', async () => { + pauseAnswers(200, { forced: false }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force: true }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('a 204 from an older API resolves to paused, not forced', async () => { + pauseAnswers(204) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('an already paused sandbox (409) resolves to not paused', async () => { + pauseAnswers(409, { code: 409, message: 'Sandbox is already paused' }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: false, forced: false }) +}) + +test('the instance method returns the same result', async () => { + server.use( + http.post(apiUrl('/v2/sandboxes/:sandboxID/connect'), () => + HttpResponse.json({ + sandboxID: 'test-sandbox-id', + templateID: 'base', + envdVersion: '0.2.4', + }) + ) + ) + pauseAnswers(200, { forced: true }) + const sandbox = await Sandbox.connect('test-sandbox-id', { + apiKey: TEST_API_KEY, + }) + + await expect(sandbox.pause({ force: true })).resolves.toEqual({ + paused: true, + forced: true, + }) +}) + +test('a refused pause (503 sandbox_pause_refused) is a SandboxPauseRefusedError carrying the API message, after one request', async () => { + const message = + "Sandbox 'test-sandbox-id' cannot be paused right now because the sandbox guest agent is not responding: retry, or pause with force to take the snapshot without the health check" + pauseAnswers(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message, + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxPauseRefusedError) + // Subclass of the generic 503 error, so handlers written for it keep working. + expect(err).toBeInstanceOf(ServiceBusyError) + expect(err.name).toBe('SandboxPauseRefusedError') + expect(err.statusCode).toBe(503) + expect(err.errorCode).toBe('sandbox_pause_refused') + // The API's message verbatim: it names the reason and what to do. + expect(err.message).toBe(message) + // A coded refusal is not replayed by the transport retries. + expect(requests).toBe(1) +}) + +test('a refusal for a busy node carries the same code and the reason in the message', async () => { + const message = + "Sandbox 'test-sandbox-id' cannot be paused right now because its node is busy, please retry" + pauseAnswers(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message, + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxPauseRefusedError) + expect(err.message).toBe(message) +}) + +test('a 503 with no error_code at all stays a plain ServiceBusyError', async () => { + pauseAnswers(503, { + code: 503, + message: + "Sandbox 'test-sandbox-id' cannot be paused right now because the server is shutting down, please retry", + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(ServiceBusyError) + expect(err).not.toBeInstanceOf(SandboxPauseRefusedError) + expect(err.message).toContain('server is shutting down') +}) + +test('force on a filesystem-only pause (400) is a SandboxError carrying the status and message', async () => { + pauseAnswers(400, { + code: 400, + message: 'force applies to memory pauses only', + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + mode: 'filesystem', + force: true, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxError) + expect(err.statusCode).toBe(400) + expect(err.message).toContain('force applies to memory pauses only') +}) + +test('a missing sandbox (404) is a SandboxNotFoundError', async () => { + pauseAnswers(404, { code: 404, message: 'Sandbox not found' }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).rejects.toBeInstanceOf(SandboxNotFoundError) +}) diff --git a/packages/python-sdk/e2b/__init__.py b/packages/python-sdk/e2b/__init__.py index b4f1d0984a..7afac418cc 100644 --- a/packages/python-sdk/e2b/__init__.py +++ b/packages/python-sdk/e2b/__init__.py @@ -38,6 +38,7 @@ ) from .volume.connection_config import VolumeApiParams, VolumeConnectionConfig from .exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, AuthenticationException, FileNotFoundException, GitAuthException, @@ -50,6 +51,7 @@ RateLimitException, ServiceBusyException, SandboxException, + SandboxPauseRefusedException, SandboxNotFoundException, SandboxNotRunningException, SandboxUnreachableException, @@ -93,6 +95,7 @@ SandboxMetrics, SandboxLifecycle, SandboxOnTimeout, + SandboxPauseResult, SandboxNetworkInfo, SandboxNetworkOpts, SandboxNetworkRule, @@ -184,6 +187,8 @@ # Exceptions "SandboxException", "ServiceBusyException", + "SandboxPauseRefusedException", + "SANDBOX_PAUSE_REFUSED_ERROR_CODE", "TimeoutException", "NotFoundException", "FileNotFoundException", @@ -258,6 +263,8 @@ "AsyncSecretPaginator", "SecretException", "SecretNotFoundException", + # Pause + "SandboxPauseResult", # Snapshot "SnapshotInfo", "SnapshotMode", diff --git a/packages/python-sdk/e2b/api/__init__.py b/packages/python-sdk/e2b/api/__init__.py index 68f7d9f243..a58d89ae7e 100644 --- a/packages/python-sdk/e2b/api/__init__.py +++ b/packages/python-sdk/e2b/api/__init__.py @@ -21,9 +21,11 @@ from e2b.api.metadata import default_headers from e2b.connection_config import ConnectionConfig, ProxyTypes from e2b.exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, AuthenticationException, InvalidArgumentException, RateLimitException, + SandboxPauseRefusedException, ServiceBusyException, SandboxException, ) @@ -157,10 +159,13 @@ def api_exception_from_code( message: Optional[str] = None, default_exception_class: type[Exception] = SandboxException, stack_trace: Optional[TracebackType] = None, + error_code: Optional[str] = None, ) -> Exception: """Map an API error code and message to the matching exception class — the same mapping :func:`handle_api_exception` applies to HTTP responses, usable - for error objects embedded in response bodies (e.g. per-fork results).""" + for error objects embedded in response bodies (e.g. per-fork results). + ``error_code`` is the body's semantic error code, which refines the status + for the codes the SDK types (a refused pause).""" if status_code == 401: text = f"{status_code}: Unauthorized, please check your credentials." if message: @@ -174,6 +179,11 @@ def api_exception_from_code( return RateLimitException(text) if status_code == 503: + if error_code == SANDBOX_PAUSE_REFUSED_ERROR_CODE: + # The API's message already says what to do for its reason. + return SandboxPauseRefusedException( + message or "The node refused the pause, please retry" + ) text = f"{status_code}: Service temporarily unavailable, please retry." if message: text += f" - {message}" @@ -197,8 +207,12 @@ def handle_api_exception( body = json.loads(e.content) if e.content else {} except json.JSONDecodeError: body = {} + # A non-object body (e.g. a JSON list) carries neither field. + if not isinstance(body, dict): + body = {} message = body["message"] if "message" in body else None + error_code = body.get("error_code") if message is None and e.status_code not in (401, 429, 503): err = default_exception_class(f"{e.status_code}: {e.content}").with_traceback( stack_trace @@ -209,7 +223,11 @@ def handle_api_exception( return err return api_exception_from_code( - e.status_code, message, default_exception_class, stack_trace + e.status_code, + message, + default_exception_class, + stack_trace, + error_code if isinstance(error_code, str) else None, ) diff --git a/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py b/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py index f0989b19e5..cfc65902ea 100644 --- a/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py +++ b/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py @@ -1,5 +1,5 @@ from http import HTTPStatus -from typing import Any, Optional, Union, cast +from typing import Any, Optional, Union import httpx @@ -7,6 +7,7 @@ from ...client import AuthenticatedClient, Client from ...models.error import Error from ...models.sandbox_pause_request import SandboxPauseRequest +from ...models.sandbox_pause_result import SandboxPauseResult from ...types import Response @@ -32,10 +33,15 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[Union[Any, Error]]: - if response.status_code == 204: - response_204 = cast(Any, None) - return response_204 +) -> Optional[Union[Error, SandboxPauseResult]]: + if response.status_code == 200: + response_200 = SandboxPauseResult.from_dict(response.json()) + + return response_200 + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 if response.status_code == 401: response_401 = Error.from_dict(response.json()) @@ -68,7 +74,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -82,7 +88,7 @@ def sync_detailed( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -96,7 +102,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[Union[Any, Error]] + Response[Union[Error, SandboxPauseResult]] """ kwargs = _get_kwargs( @@ -116,7 +122,7 @@ def sync( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Optional[Union[Any, Error]]: +) -> Optional[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -130,7 +136,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Union[Any, Error] + Union[Error, SandboxPauseResult] """ return sync_detailed( @@ -145,7 +151,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -159,7 +165,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[Union[Any, Error]] + Response[Union[Error, SandboxPauseResult]] """ kwargs = _get_kwargs( @@ -177,7 +183,7 @@ async def asyncio( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Optional[Union[Any, Error]]: +) -> Optional[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -191,7 +197,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Union[Any, Error] + Union[Error, SandboxPauseResult] """ return ( diff --git a/packages/python-sdk/e2b/api/client/models/__init__.py b/packages/python-sdk/e2b/api/client/models/__init__.py index 8aba151880..1d25d5eca2 100644 --- a/packages/python-sdk/e2b/api/client/models/__init__.py +++ b/packages/python-sdk/e2b/api/client/models/__init__.py @@ -50,6 +50,7 @@ from .sandbox_network_update_config_rules import SandboxNetworkUpdateConfigRules from .sandbox_on_timeout import SandboxOnTimeout from .sandbox_pause_request import SandboxPauseRequest +from .sandbox_pause_result import SandboxPauseResult from .sandbox_refresh_request import SandboxRefreshRequest from .sandbox_snapshot_request import SandboxSnapshotRequest from .sandbox_state import SandboxState @@ -132,6 +133,7 @@ "SandboxNetworkUpdateConfigRules", "SandboxOnTimeout", "SandboxPauseRequest", + "SandboxPauseResult", "SandboxRefreshRequest", "SandboxSnapshotRequest", "SandboxState", diff --git a/packages/python-sdk/e2b/api/client/models/error.py b/packages/python-sdk/e2b/api/client/models/error.py index a0a99778ac..7739540e8d 100644 --- a/packages/python-sdk/e2b/api/client/models/error.py +++ b/packages/python-sdk/e2b/api/client/models/error.py @@ -17,6 +17,8 @@ class Error: message (str): Error error_code (Union[Unset, str]): Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, + sandbox_pause_refused (the node refused the pause and the sandbox keeps running: the guest agent did not answer + the health check, or the node was busy; retry, or pause with force when the guest agent is unresponsive), internal_server_error, secret_limit_reached. """ diff --git a/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py b/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py index 0f36b2eb83..06c4164e6d 100644 --- a/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py +++ b/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py @@ -17,19 +17,27 @@ class SandboxPauseRequest: persisted and resuming the sandbox cold-boots (reboots) it from disk, losing in-memory state, running processes, and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. Default: True. + force (Union[Unset, bool]): Take the pause even when the sandbox guest agent is unresponsive. Applies to memory + pauses only. The memory snapshot may not resume; a reboot resume recovers the disk state. Defaults to false. + Default: False. """ memory: Union[Unset, bool] = True + force: Union[Unset, bool] = False additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: memory = self.memory + force = self.force + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) if memory is not UNSET: field_dict["memory"] = memory + if force is not UNSET: + field_dict["force"] = force return field_dict @@ -38,8 +46,11 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: d = dict(src_dict) memory = d.pop("memory", UNSET) + force = d.pop("force", UNSET) + sandbox_pause_request = cls( memory=memory, + force=force, ) sandbox_pause_request.additional_properties = d diff --git a/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py b/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py new file mode 100644 index 0000000000..cd26ee7d33 --- /dev/null +++ b/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="SandboxPauseResult") + + +@_attrs_define +class SandboxPauseResult: + """ + Attributes: + forced (bool): Whether the pause was taken with the force override, skipping the guest agent health check. + """ + + forced: bool + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + forced = self.forced + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "forced": forced, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + forced = d.pop("forced") + + sandbox_pause_result = cls( + forced=forced, + ) + + sandbox_pause_result.additional_properties = d + return sandbox_pause_result + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index d479e21313..914a2a7790 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -223,6 +223,37 @@ class ServiceBusyException(Exception): status_code = 503 +# ``error_code`` of the 503 the API answers a pause with when the node refused +# it. Any endpoint answering this code is mapped to +# ``SandboxPauseRefusedException``. +SANDBOX_PAUSE_REFUSED_ERROR_CODE = "sandbox_pause_refused" + + +class SandboxPauseRefusedException(ServiceBusyException): + """ + Raised when the node refused a pause (HTTP 503 with + ``error_code: "sandbox_pause_refused"``). The sandbox keeps running with its + state intact; the message is the API's, and names the reason and what to + do about it. In general: retry, and when the reason is an unresponsive + sandbox guest agent, pause with ``force=True`` to take the snapshot without + the health check -- its memory may then not resume, and a resume with + ``on_resume="reboot"`` recovers the disk state. + + Unlike other 503s, a refusal carrying this code is not replayed by the + SDK's transport retries: it is a decision, not transient capacity, so it + surfaces after a single request and the caller decides what to do. + + Subclass of `ServiceBusyException`, which a refused pause surfaced as + before, so existing handlers keep working. Unlike the generic 503 mapping, + the message carries no ``503: `` prefix: it is the API's message verbatim. + + The ``error_code`` attribute is the semantic error code of the API + response, always ``sandbox_pause_refused``. + """ + + error_code = SANDBOX_PAUSE_REFUSED_ERROR_CODE + + class BuildException(Exception): """ Raised when the build fails. diff --git a/packages/python-sdk/e2b/retry.py b/packages/python-sdk/e2b/retry.py index 50510fccfa..5e8b0f1560 100644 --- a/packages/python-sdk/e2b/retry.py +++ b/packages/python-sdk/e2b/retry.py @@ -1,12 +1,25 @@ import asyncio +import json import random import re import time -from typing import Awaitable, Callable, List, Optional, Tuple +from dataclasses import dataclass, field +from collections.abc import AsyncIterator, Iterator +from typing import ( + Awaitable, + Callable, + List, + Optional, + Tuple, + Union, +) import httpx -from e2b.exceptions import InvalidArgumentException +from e2b.exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, + InvalidArgumentException, +) MAX_RETRY_AFTER_SECONDS = 2_147_483_647 MAX_RETRY_WAIT_WITHOUT_TIMEOUT_SECONDS = 60.0 @@ -14,6 +27,16 @@ BACKOFF_MAX_SECONDS = 10.0 BACKOFF_JITTER_MIN = 0.5 RETRYABLE_STATUSES = frozenset({429, 502, 503}) +# A 503 whose body carries one of these ``error_code`` values is a decision by +# the server, not transient capacity, and is answered after a single request. +NON_RETRYABLE_ERROR_CODES = frozenset({SANDBOX_PAUSE_REFUSED_ERROR_CODE}) +# An error body larger than this cannot carry a code worth parsing, so the peek +# stops there: such a 503 is retried while budget remains and handed on whole +# when it is exhausted. +MAX_ERROR_BODY_BYTES = 8 * 1024 +# Wall-clock cap on peeking at a 503 body for its ``error_code``, measured +# across chunks; the JavaScript transport's PEEK_BUDGET_MS counterpart. +PEEK_BUDGET_SECONDS = 1.0 # GET/PUT/PATCH/DELETE are idempotent by HTTP semantics. A POST is retried # after a 502 or a network error that may have occurred once the request was @@ -104,6 +127,263 @@ def _retry_delay( return _backoff_delay(attempt, random_) +def _media_type(content_type: Optional[str]) -> str: + """The media type of a ``Content-Type`` value: lower-cased, parameters + dropped.""" + return (content_type or "").split(";", 1)[0].strip().lower() + + +def _should_read_error_body(response: httpx.Response) -> bool: + """Whether a 503 body is worth reading for its ``error_code``: the status + matches, the body is JSON, and it is not declared larger than + ``MAX_ERROR_BODY_BYTES``. The read itself streams up to that cap (so a + chunked or HTTP/2 body without ``Content-Length`` is read too). Each chunk + is bounded by the request's read timeout as the transport applies it + (pyqwest's operation timeout; none when the request timeout is disabled), + and the whole peek by a wall-clock budget measured across chunks (see + ``_read_error_body``). A read that fails, exceeds the cap or outlives the + budget decides nothing: the 503 is retried while budget remains; with none + left, a body past the cap or cut off by the budget is handed on whole and + a failed read is re-raised.""" + if response.status_code != 503: + return False + if _media_type(response.headers.get("Content-Type")) != "application/json": + return False + length = response.headers.get("Content-Length") + return not ( + length is not None and length.isdecimal() and int(length) > MAX_ERROR_BODY_BYTES + ) + + +@dataclass +class _Peek: + """What peeking at a 503 body yielded: ``content`` is the whole body when + it was read in full; otherwise ``buffered`` holds the chunks read before + the peek stopped (at the byte cap or past the wall-clock budget) and + ``rest`` the live iterator over the remainder, so the body can be handed + on whole or released.""" + + content: Optional[bytes] = None + buffered: List[bytes] = field(default_factory=list) + rest: Union[Iterator[bytes], AsyncIterator[bytes], None] = None + + +def _close_rest(peek: _Peek) -> None: + """Release a peek's remaining body iterator (httpx hands out a generator) + now rather than at garbage collection. A generator a reader is still + running cannot be closed from outside; it is left to that reader.""" + close = getattr(peek.rest, "close", None) + if close is not None: + try: + close() + except (RuntimeError, ValueError): + # "generator already executing": a reader owns it. + pass + + +async def _aclose_rest(peek: _Peek) -> None: + """Async counterpart of :func:`_close_rest`.""" + aclose = getattr(peek.rest, "aclose", None) + if aclose is not None: + try: + await aclose() + except (RuntimeError, ValueError): + # "asynchronous generator is already running": a reader owns it. + pass + + +def _read_error_body( + response: httpx.Response, until: float, monotonic: Callable[[], float] +) -> _Peek: + """Up to ``MAX_ERROR_BODY_BYTES`` of the (decoded) body. Stops, keeping + the stream resumable, at the byte cap or once a wait for a further chunk + began after ``until`` (``monotonic()``). A chunk whose wait began within + the budget is kept whenever it arrives, so a complete body is never + dropped for being slow; only waiting for more past the budget gives up.""" + chunks: List[bytes] = [] + size = 0 + stream = response.iter_bytes() + while True: + waited_from = monotonic() + chunk = next(stream, None) + if chunk is None: + return _Peek(content=b"".join(chunks)) + chunks.append(chunk) + size += len(chunk) + if size > MAX_ERROR_BODY_BYTES or waited_from > until: + return _Peek(buffered=chunks, rest=stream) + + +async def _aread_error_body( + response: httpx.Response, until: float, monotonic: Callable[[], float] +) -> _Peek: + """Async counterpart of :func:`_read_error_body`.""" + chunks: List[bytes] = [] + size = 0 + stream = response.aiter_bytes() + while True: + waited_from = monotonic() + try: + chunk = await stream.__anext__() + except StopAsyncIteration: + return _Peek(content=b"".join(chunks)) + chunks.append(chunk) + size += len(chunk) + if size > MAX_ERROR_BODY_BYTES or waited_from > until: + return _Peek(buffered=chunks, rest=stream) + + +_TAIL_TIMEOUT_MESSAGE = ( + "Request timed out while reading the response body. " + "Increase `request_timeout` or lower `retries`." +) + + +class _ResumedStream(httpx.SyncByteStream): + """The peeked chunks followed by the rest of the original body, the tail + bounded by the request deadline: a chunk whose wait would begin past it + is not waited for; the inner response is closed and ``ReadTimeout`` is + raised, so the client's read ends at the deadline instead of running on + chunk by chunk. Since the peek's time counts against the retry delay, a + budget exhausted after a peek means the clock is at the deadline, so in + practice the caller gets what the peek buffered and then the timeout; a + body the peek read in full, or buffered chunks that already form the + refusal, are returned as content instead and never reach this stream.""" + + def __init__( + self, + response: httpx.Response, + peek: _Peek, + request: httpx.Request, + deadline: float, + monotonic: Callable[[], float], + ): + self._response = response + self._peek = peek + self._request = request + self._deadline = deadline + self._monotonic = monotonic + + def __iter__(self) -> Iterator[bytes]: + yield from self._peek.buffered + rest = self._peek.rest + if not isinstance(rest, Iterator): + return + while True: + if self._monotonic() >= self._deadline: + self.close() + raise httpx.ReadTimeout(_TAIL_TIMEOUT_MESSAGE, request=self._request) + chunk = next(rest, None) + if chunk is None: + return + yield chunk + + def close(self) -> None: + try: + _close_rest(self._peek) + finally: + self._response.close() + + +class _AsyncResumedStream(httpx.AsyncByteStream): + def __init__( + self, + response: httpx.Response, + peek: _Peek, + request: httpx.Request, + deadline: float, + monotonic: Callable[[], float], + ): + self._response = response + self._peek = peek + self._request = request + self._deadline = deadline + self._monotonic = monotonic + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._peek.buffered: + yield chunk + rest = self._peek.rest + if not isinstance(rest, AsyncIterator): + return + while True: + if self._monotonic() >= self._deadline: + await self.aclose() + raise httpx.ReadTimeout(_TAIL_TIMEOUT_MESSAGE, request=self._request) + try: + chunk = await rest.__anext__() + except StopAsyncIteration: + return + yield chunk + + async def aclose(self) -> None: + try: + await _aclose_rest(self._peek) + finally: + await self._response.aclose() + + +# Response extensions that are plain values and survive rebuilding the response; +# pyqwest's ``get_trailers`` is bound to the native response and is not copied. +_REBUILT_EXTENSIONS = ("http_version", "reason_phrase") + + +def _rebuild( + response: httpx.Response, + request: httpx.Request, + *, + content: Optional[bytes] = None, + stream: Union[httpx.SyncByteStream, httpx.AsyncByteStream, None] = None, +) -> httpx.Response: + """The response with its (partly) consumed stream replaced by ``content`` + or ``stream`` (httpx takes the stream when both are given), both decoded, + so the encoding, transfer-encoding and length headers are dropped (httpx + sets the length of a content body). ``request`` is the one being handled: + the transport's response carries none yet, httpx attaches it after the + transport returns.""" + headers = [ + (name, value) + for name, value in response.headers.raw + if name.lower() + not in (b"content-encoding", b"transfer-encoding", b"content-length") + ] + return httpx.Response( + response.status_code, + headers=headers, + content=content, + stream=stream, + request=request, + extensions={ + key: value + for key, value in response.extensions.items() + if key in _REBUILT_EXTENSIONS + }, + ) + + +def _is_complete_json(content: bytes) -> bool: + """Whether ``content`` parses as one complete JSON document.""" + try: + json.loads(content) + except (ValueError, RecursionError): + return False + return True + + +def _is_non_retryable_refusal(content: bytes) -> bool: + """Whether a 503 body carries a non-retryable ``error_code`` (see + ``NON_RETRYABLE_ERROR_CODES``).""" + try: + body = json.loads(content) + except (ValueError, RecursionError): + # Not JSON, or nested too deep to parse: not a refusal either way. + return False + if not isinstance(body, dict): + return False + code = body.get("error_code") + return isinstance(code, str) and code in NON_RETRYABLE_ERROR_CODES + + def _copy_request( request: httpx.Request, remaining_timeout: Optional[float] = None ) -> httpx.Request: @@ -142,7 +422,9 @@ class RetryableTransport(httpx.BaseTransport): """Retry requests after a 429 carrying ``Retry-After`` or a 503, and — for replayable operations (``is_replayable``) — after a 502 (using ``Retry-After`` when present, exponential backoff otherwise) or a network - error once the request was written.""" + error once the request was written. A 503 whose body carries a + non-retryable ``error_code`` (a refused pause) is returned after one + request.""" def __init__( self, @@ -196,6 +478,63 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: delay = _retry_delay(response, attempt, replayable, self._random) if delay is None or self._monotonic() + delay >= deadline: return response + if _should_read_error_body(response): + read_started = self._monotonic() + peek_until = min( + read_started + max(delay, PEEK_BUDGET_SECONDS), deadline + ) + peek = _Peek() + read_error: Optional[Exception] = None + try: + peek = _read_error_body(response, peek_until, self._monotonic) + except (httpx.HTTPError, httpx.StreamError) as exc: + # A body that cannot be read decides nothing: retry as before. + read_error = exc + except BaseException: + # Interrupted mid-peek: do not leak the connection. + response.close() + raise + if peek.content is None and peek.rest is not None: + # Cut off on what was the last chunk: the buffered bytes are one + # complete JSON document, so they are the body read in full (a + # refusal or any other complete API error) and the tail is left + # unread; only a body that does not yet parse is handed on. + buffered = b"".join(peek.buffered) + if len(buffered) <= MAX_ERROR_BODY_BYTES and _is_complete_json( + buffered + ): + _close_rest(peek) + peek = _Peek(content=buffered) + if peek.content is not None and _is_non_retryable_refusal(peek.content): + response.close() + return _rebuild(response, request, content=peek.content) + # The time spent reading counts towards the delay. + delay = max(0.0, delay - (self._monotonic() - read_started)) + if self._monotonic() + delay >= deadline: + # Out of budget: the 503 is the outcome, as on the + # JavaScript side. A body read in full is returned as is; + # one stopped at the byte cap or cut off by the peek budget + # is handed on whole, the peeked chunks first and the rest + # of the stream after; a failed read is reported as such, + # never as a fabricated body (the API client parses every + # 503 body as JSON). + if peek.content is not None: + response.close() + return _rebuild(response, request, content=peek.content) + if peek.rest is not None: + return _rebuild( + response, + request, + stream=_ResumedStream( + response, peek, request, deadline, self._monotonic + ), + ) + response.close() + if read_error is not None: + raise read_error + raise AssertionError("unreachable") + # Retrying: release what the peek left of the body. + _close_rest(peek) response.close() self._sleep(delay) @@ -262,6 +601,66 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: delay = _retry_delay(response, attempt, replayable, self._random) if delay is None or self._monotonic() + delay >= deadline: return response + if _should_read_error_body(response): + read_started = self._monotonic() + peek_until = min( + read_started + max(delay, PEEK_BUDGET_SECONDS), deadline + ) + peek = _Peek() + read_error: Optional[Exception] = None + try: + peek = await _aread_error_body( + response, peek_until, self._monotonic + ) + except (httpx.HTTPError, httpx.StreamError) as exc: + # A body that cannot be read decides nothing: retry as before. + read_error = exc + except BaseException: + # Interrupted mid-peek (e.g. cancelled): do not leak the + # connection. + await response.aclose() + raise + if peek.content is None and peek.rest is not None: + # Cut off on what was the last chunk: the buffered bytes are one + # complete JSON document, so they are the body read in full (a + # refusal or any other complete API error) and the tail is left + # unread; only a body that does not yet parse is handed on. + buffered = b"".join(peek.buffered) + if len(buffered) <= MAX_ERROR_BODY_BYTES and _is_complete_json( + buffered + ): + await _aclose_rest(peek) + peek = _Peek(content=buffered) + if peek.content is not None and _is_non_retryable_refusal(peek.content): + await response.aclose() + return _rebuild(response, request, content=peek.content) + # The time spent reading counts towards the delay. + delay = max(0.0, delay - (self._monotonic() - read_started)) + if self._monotonic() + delay >= deadline: + # Out of budget: the 503 is the outcome, as on the + # JavaScript side. A body read in full is returned as is; + # one stopped at the byte cap or cut off by the peek budget + # is handed on whole, the peeked chunks first and the rest + # of the stream after; a failed read is reported as such, + # never as a fabricated body (the API client parses every + # 503 body as JSON). + if peek.content is not None: + await response.aclose() + return _rebuild(response, request, content=peek.content) + if peek.rest is not None: + return _rebuild( + response, + request, + stream=_AsyncResumedStream( + response, peek, request, deadline, self._monotonic + ), + ) + await response.aclose() + if read_error is not None: + raise read_error + raise AssertionError("unreachable") + # Retrying: release what the peek left of the body. + await _aclose_rest(peek) await response.aclose() await self._sleep(delay) diff --git a/packages/python-sdk/e2b/sandbox/sandbox_api.py b/packages/python-sdk/e2b/sandbox/sandbox_api.py index 7b75c8d3f3..868eb2e026 100644 --- a/packages/python-sdk/e2b/sandbox/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox/sandbox_api.py @@ -573,6 +573,24 @@ def resolve_snapshot_memory( return mode == "full" +@dataclass +class SandboxPauseResult: + """ + Result of a pause. + """ + + paused: bool + """`True` if this call paused the sandbox, `False` if it was already paused.""" + + forced: bool + """ + Whether the snapshot was taken with the `force` override, without the guest + agent health check. Reports what happened rather than echoing the option: + `False` when the sandbox was already paused, and also when a forced request + joined a normal pause that was already in flight. + """ + + class SandboxOnTimeoutPause(TypedDict): """ Object form of `on_timeout` that auto-pauses the sandbox when the timeout is diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py index 3d0a648f50..a2723f9e56 100644 --- a/packages/python-sdk/e2b/sandbox_async/main.py +++ b/packages/python-sdk/e2b/sandbox_async/main.py @@ -31,6 +31,7 @@ SandboxNetworkUpdate, SandboxOnResume, SnapshotInfo, + SandboxPauseResult, SnapshotMode, ) from e2b.sandbox.utils import class_method_variant @@ -789,15 +790,19 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -808,16 +813,20 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox specified by sandbox ID. :param sandbox_id: Sandbox ID :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -827,21 +836,26 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ return await SandboxApi._cls_pause( sandbox_id=self.sandbox_id, keep_memory=keep_memory, mode=mode, + force=force, **self.connection_config.get_api_params(**opts), ) @@ -851,8 +865,9 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @overload @staticmethod @@ -861,8 +876,9 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @class_method_variant("_cls_pause") async def beta_pause( @@ -870,14 +886,15 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ :deprecated: Use `pause()` instead. - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) """ - return await self.pause(keep_memory=keep_memory, mode=mode, **opts) + return await self.pause(keep_memory=keep_memory, mode=mode, force=force, **opts) @overload async def create_snapshot( diff --git a/packages/python-sdk/e2b/sandbox_async/sandbox_api.py b/packages/python-sdk/e2b/sandbox_async/sandbox_api.py index e41f8ff397..d735c2dc2c 100644 --- a/packages/python-sdk/e2b/sandbox_async/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox_async/sandbox_api.py @@ -33,6 +33,7 @@ SandboxForkRequest, SandboxNetworkConfig, SandboxPauseRequest, + SandboxPauseResult as SandboxPauseResultAPI, SandboxVolumeMount as SandboxVolumeMountAPI, ) from e2b.api.client.types import UNSET, Unset @@ -56,6 +57,7 @@ SandboxNetworkOpts, SandboxNetworkUpdate, SandboxOnResume, + SandboxPauseResult, SnapshotMode, resolve_connect_memory, resolve_snapshot_memory, @@ -400,8 +402,9 @@ async def _cls_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: config = ConnectionConfig(**cls._resolve_api_params(**opts)) api_client = get_api_client(config) @@ -409,7 +412,8 @@ async def _cls_pause( sandbox_id, client=api_client, body=SandboxPauseRequest( - memory=resolve_snapshot_memory("mode", mode, keep_memory) + memory=resolve_snapshot_memory("mode", mode, keep_memory), + force=UNSET if force is None else force, ), ) @@ -418,7 +422,7 @@ async def _cls_pause( if res.status_code == 409: # Sandbox is already paused - return False + return SandboxPauseResult(paused=False, forced=False) if res.status_code >= 300: raise handle_api_exception(res) @@ -427,7 +431,13 @@ async def _cls_pause( if isinstance(res.parsed, Error): raise SandboxException(f"{res.parsed.message}: Request failed") - return True + # An older API answers 204 without a body. + forced = ( + res.parsed.forced + if isinstance(res.parsed, SandboxPauseResultAPI) + else False + ) + return SandboxPauseResult(paused=True, forced=forced) @classmethod async def _cls_fork( @@ -484,7 +494,15 @@ async def _cls_fork( # would have failed the whole request — so stay generic. exception = NotFoundException(f"{error.code}: {error.message}") else: - exception = api_exception_from_code(error.code, error.message) + exception = api_exception_from_code( + error.code, + error.message, + error_code=( + error.error_code + if isinstance(error.error_code, str) + else None + ), + ) results.append(exception) continue diff --git a/packages/python-sdk/e2b/sandbox_sync/main.py b/packages/python-sdk/e2b/sandbox_sync/main.py index 32c1b671b7..bfa50db984 100644 --- a/packages/python-sdk/e2b/sandbox_sync/main.py +++ b/packages/python-sdk/e2b/sandbox_sync/main.py @@ -30,6 +30,7 @@ SandboxNetworkUpdate, SandboxOnResume, SnapshotInfo, + SandboxPauseResult, SnapshotMode, ) from e2b.sandbox.utils import class_method_variant @@ -787,15 +788,19 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -806,16 +811,20 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox specified by sandbox ID. :param sandbox_id: Sandbox ID :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -825,21 +834,26 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ return SandboxApi._cls_pause( sandbox_id=self.sandbox_id, keep_memory=keep_memory, mode=mode, + force=force, **self.connection_config.get_api_params(**opts), ) @@ -849,8 +863,9 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @overload @staticmethod @@ -859,8 +874,9 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @class_method_variant("_cls_pause") def beta_pause( @@ -868,14 +884,15 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ :deprecated: Use `pause()` instead. - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) """ - return self.pause(keep_memory=keep_memory, mode=mode, **opts) + return self.pause(keep_memory=keep_memory, mode=mode, force=force, **opts) @overload def create_snapshot( diff --git a/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py b/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py index 8f4db3e890..82d30cc142 100644 --- a/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py @@ -33,6 +33,7 @@ SandboxForkRequest, SandboxNetworkConfig, SandboxPauseRequest, + SandboxPauseResult as SandboxPauseResultAPI, SandboxVolumeMount as SandboxVolumeMountAPI, ) from e2b.api.client.types import UNSET, Unset @@ -55,6 +56,7 @@ SandboxNetworkOpts, SandboxNetworkUpdate, SandboxOnResume, + SandboxPauseResult, SnapshotMode, resolve_connect_memory, resolve_snapshot_memory, @@ -413,7 +415,15 @@ def _cls_fork( # would have failed the whole request — so stay generic. exception = NotFoundException(f"{error.code}: {error.message}") else: - exception = api_exception_from_code(error.code, error.message) + exception = api_exception_from_code( + error.code, + error.message, + error_code=( + error.error_code + if isinstance(error.error_code, str) + else None + ), + ) results.append(exception) continue @@ -488,8 +498,9 @@ def _cls_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: config = ConnectionConfig(**cls._resolve_api_params(**opts)) api_client = get_api_client(config) @@ -497,7 +508,8 @@ def _cls_pause( sandbox_id, client=api_client, body=SandboxPauseRequest( - memory=resolve_snapshot_memory("mode", mode, keep_memory) + memory=resolve_snapshot_memory("mode", mode, keep_memory), + force=UNSET if force is None else force, ), ) @@ -506,7 +518,7 @@ def _cls_pause( if res.status_code == 409: # Sandbox is already paused - return False + return SandboxPauseResult(paused=False, forced=False) if res.status_code >= 300: raise handle_api_exception(res) @@ -515,4 +527,10 @@ def _cls_pause( if isinstance(res.parsed, Error): raise SandboxException(f"{res.parsed.message}: Request failed") - return True + # An older API answers 204 without a body. + forced = ( + res.parsed.forced + if isinstance(res.parsed, SandboxPauseResultAPI) + else False + ) + return SandboxPauseResult(paused=True, forced=forced) diff --git a/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py b/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py index 7a1ecfacda..66f1ee07b9 100644 --- a/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py +++ b/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py @@ -13,7 +13,7 @@ async def test_pause_filesystem_only(async_sandbox: AsyncSandbox): ).strip() # Filesystem-only pause: only the rootfs is persisted, no memory snapshot. - assert await async_sandbox.pause(mode="filesystem", request_timeout=120) + assert (await async_sandbox.pause(mode="filesystem", request_timeout=120)).paused assert not await async_sandbox.is_running() # Resuming a filesystem-only snapshot cold-boots (reboots) from the rootfs. diff --git a/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py b/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py index 6945e35f4e..78c2940514 100644 --- a/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py +++ b/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py @@ -187,8 +187,12 @@ async def test_async_fork_rejects_count_outside_1_to_20( request.assert_not_called() +def _paused(): + return SimpleNamespace(status_code=200, parsed=SimpleNamespace(forced=False)) + + def _sync_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: - request = Mock(return_value=SimpleNamespace(status_code=204, parsed=None)) + request = Mock(return_value=_paused()) monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "sync_detailed", request) Sandbox.pause("sbx-test", api_key=api_key, **kwargs) @@ -197,7 +201,7 @@ def _sync_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: async def _async_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: - request = AsyncMock(return_value=SimpleNamespace(status_code=204, parsed=None)) + request = AsyncMock(return_value=_paused()) monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "asyncio_detailed", request) await AsyncSandbox.pause("sbx-test", api_key=api_key, **kwargs) @@ -227,6 +231,26 @@ def test_pause_still_sends_the_deprecated_keep_memory(monkeypatch, test_api_key) assert body["memory"] is False +def test_pause_omits_force_when_unset(monkeypatch, test_api_key): + body = _sync_pause_body(monkeypatch, test_api_key) + + assert "force" not in body + + +@pytest.mark.parametrize("force", [True, False]) +def test_pause_sends_an_explicit_force(monkeypatch, test_api_key, force): + body = _sync_pause_body(monkeypatch, test_api_key, force=force) + + assert body["force"] is force + + +def test_pause_sends_force_alongside_mode(monkeypatch, test_api_key): + body = _sync_pause_body(monkeypatch, test_api_key, mode="full", force=True) + + assert body["memory"] is True + assert body["force"] is True + + @pytest.mark.parametrize( "kwargs, match", [ @@ -265,6 +289,19 @@ async def test_async_pause_still_sends_the_deprecated_keep_memory( assert body["memory"] is False +async def test_async_pause_omits_force_when_unset(monkeypatch, test_api_key): + body = await _async_pause_body(monkeypatch, test_api_key) + + assert "force" not in body + + +@pytest.mark.parametrize("force", [True, False]) +async def test_async_pause_sends_an_explicit_force(monkeypatch, test_api_key, force): + body = await _async_pause_body(monkeypatch, test_api_key, force=force) + + assert body["force"] is force + + def _snapshot_response(): return SimpleNamespace( status_code=201, diff --git a/packages/python-sdk/tests/shared/sandbox/test_pause.py b/packages/python-sdk/tests/shared/sandbox/test_pause.py new file mode 100644 index 0000000000..a3e15ba28e --- /dev/null +++ b/packages/python-sdk/tests/shared/sandbox/test_pause.py @@ -0,0 +1,756 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import httpx + +import pytest + +from e2b import ( + AsyncSandbox, + TimeoutException, + Sandbox, + SandboxPauseResult, + SandboxException, + SandboxNotFoundException, + SandboxPauseRefusedException, + ServiceBusyException, +) +import e2b.sandbox_async.sandbox_api as async_sandbox_api +import e2b.sandbox_sync.sandbox_api as sync_sandbox_api +from e2b.api import ApiClient, AsyncApiClient +from e2b.retry import AsyncRetryableTransport, RetryableTransport +from e2b.api.client.api.sandboxes import post_sandboxes_sandbox_id_pause +from e2b.api.client.models import Error +from e2b.api.client.models import SandboxPauseResult as SandboxPauseResultAPI + +REFUSED_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because the sandbox guest " + "agent is not responding: retry, or pause with force to take the snapshot " + "without the health check" +) +BUSY_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because its node is busy, " + "please retry" +) +DRAINING_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because the server is " + "shutting down, please retry" +) + + +def _response(status_code: int, body: dict): + """A pause response the way the generated client builds it: the raw body + and, for an error status, the parsed ``Error`` model.""" + parsed = ( + SandboxPauseResultAPI.from_dict(body) + if status_code == 200 + else Error.from_dict(body) + ) + return SimpleNamespace( + status_code=status_code, + content=json.dumps(body).encode(), + parsed=parsed, + headers={}, + ) + + +def _answer(monkeypatch, response): + request = Mock(return_value=response) + monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "sync_detailed", request) + return request + + +def _answer_async(monkeypatch, response): + request = AsyncMock(return_value=response) + monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "asyncio_detailed", request) + return request + + +def test_pause_returns_paused_not_forced(monkeypatch, test_api_key): + _answer(monkeypatch, _response(200, {"forced": False})) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=True, forced=False + ) + + +def test_forced_pause_returns_paused_and_forced(monkeypatch, test_api_key): + request = _answer(monkeypatch, _response(200, {"forced": True})) + + assert Sandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=True) + assert request.call_args.kwargs["body"].to_dict()["force"] is True + + +def test_forced_reports_what_the_api_did_not_the_option(monkeypatch, test_api_key): + # A forced request that joined a normal pause already in flight is not forced. + _answer(monkeypatch, _response(200, {"forced": False})) + + assert Sandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +def test_pause_on_an_older_api_answering_204_is_paused_not_forced( + monkeypatch, test_api_key +): + _answer(monkeypatch, SimpleNamespace(status_code=204, parsed=None, content=b"")) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=True, forced=False + ) + + +def test_pause_of_an_already_paused_sandbox_is_not_paused(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response(409, {"code": 409, "message": "Sandbox is already paused"}), + ) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=False, forced=False + ) + + +def test_refused_pause_raises_the_typed_exception(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + err = excinfo.value + # Subclass of the generic 503 exception, so handlers written for it keep working. + assert isinstance(err, ServiceBusyException) + assert err.status_code == 503 + assert err.error_code == "sandbox_pause_refused" + # The API's message verbatim: it names the reason and what to do. + assert str(err) == REFUSED_MESSAGE + + +def test_refusal_for_a_busy_node_carries_the_same_code_and_the_reason( + monkeypatch, test_api_key +): + _answer( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": BUSY_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == BUSY_MESSAGE + + +def test_503_with_no_error_code_at_all_stays_a_busy_exception( + monkeypatch, test_api_key +): + _answer(monkeypatch, _response(503, {"code": 503, "message": DRAINING_MESSAGE})) + + with pytest.raises(ServiceBusyException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "server is shutting down" in str(excinfo.value) + + +def test_force_on_a_filesystem_only_pause_is_a_400(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response(400, {"code": 400, "message": "force applies to memory pauses only"}), + ) + + with pytest.raises(SandboxException) as excinfo: + Sandbox.pause("sbx-test", mode="filesystem", force=True, api_key=test_api_key) + + assert excinfo.value.status_code == 400 + assert "force applies to memory pauses only" in str(excinfo.value) + + +def test_pause_of_a_missing_sandbox_raises_not_found(monkeypatch, test_api_key): + _answer(monkeypatch, _response(404, {"code": 404, "message": "Sandbox not found"})) + + with pytest.raises(SandboxNotFoundException): + Sandbox.pause("sbx-test", api_key=test_api_key) + + +async def test_async_pause_returns_paused_not_forced(monkeypatch, test_api_key): + _answer_async(monkeypatch, _response(200, {"forced": False})) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +async def test_async_forced_pause_returns_paused_and_forced(monkeypatch, test_api_key): + request = _answer_async(monkeypatch, _response(200, {"forced": True})) + + assert await AsyncSandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=True) + assert request.call_args.kwargs["body"].to_dict()["force"] is True + + +async def test_async_pause_on_an_older_api_answering_204_is_paused_not_forced( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, SimpleNamespace(status_code=204, parsed=None, content=b"") + ) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +async def test_async_pause_of_an_already_paused_sandbox_is_not_paused( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response(409, {"code": 409, "message": "Sandbox is already paused"}), + ) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=False, forced=False) + + +async def test_async_refused_pause_raises_the_typed_exception( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert isinstance(excinfo.value, ServiceBusyException) + assert excinfo.value.status_code == 503 + assert str(excinfo.value) == REFUSED_MESSAGE + + +async def test_async_503_with_no_error_code_at_all_stays_a_busy_exception( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, _response(503, {"code": 503, "message": DRAINING_MESSAGE}) + ) + + with pytest.raises(ServiceBusyException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + + +async def test_async_force_on_a_filesystem_only_pause_is_a_400( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response(400, {"code": 400, "message": "force applies to memory pauses only"}), + ) + + with pytest.raises(SandboxException) as excinfo: + await AsyncSandbox.pause( + "sbx-test", mode="filesystem", force=True, api_key=test_api_key + ) + + assert excinfo.value.status_code == 400 + + +# The tests above mock the generated client's `sync_detailed`; these go through +# its real response parsing over an httpx mock transport, so the 200 body is +# parsed into the generated model and the 204 path sees `parsed is None`. + + +def _mock_api(monkeypatch, answer, clock=None): + """Route both clients through the SDK's own retry transports over an + httpx mock transport whose responses, like pyqwest's, carry no request + (httpx attaches it after the transport returns). ``answer`` builds a fresh + response per request; ``clock`` replaces the transports' monotonic clock + and sleep.""" + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return answer() + + def sync_client(config): + inner = httpx.MockTransport(handler) + if clock is None: + transport = RetryableTransport(inner, config.retries) + else: + transport = RetryableTransport( + inner, config.retries, sleep=clock.sleep, monotonic=clock.monotonic + ) + return ApiClient(config, transport=transport) + + def async_client(config): + inner = httpx.MockTransport(handler) + if clock is None: + transport = AsyncRetryableTransport(inner, config.retries) + else: + fake = clock + + async def async_sleep(delay: float) -> None: + fake.sleep(delay) + + transport = AsyncRetryableTransport( + inner, config.retries, sleep=async_sleep, monotonic=fake.monotonic + ) + return AsyncApiClient(config, transport=transport) + + monkeypatch.setattr(sync_sandbox_api, "get_api_client", sync_client) + monkeypatch.setattr(async_sandbox_api, "get_api_client", async_client) + return seen + + +@pytest.mark.parametrize("forced", [True, False]) +def test_pause_parses_the_200_body(monkeypatch, test_api_key, forced): + seen = _mock_api(monkeypatch, lambda: httpx.Response(200, json={"forced": forced})) + + result = Sandbox.pause("sbx-test", force=forced, api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=forced) + assert json.loads(seen[0].content) == {"force": forced} + + +def test_pause_accepts_a_204_from_an_older_api(monkeypatch, test_api_key): + seen = _mock_api(monkeypatch, lambda: httpx.Response(204)) + + result = Sandbox.pause("sbx-test", api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=False) + assert json.loads(seen[0].content) == {} + + +@pytest.mark.parametrize("forced", [True, False]) +async def test_async_pause_parses_the_200_body(monkeypatch, test_api_key, forced): + seen = _mock_api(monkeypatch, lambda: httpx.Response(200, json={"forced": forced})) + + result = await AsyncSandbox.pause("sbx-test", force=forced, api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=forced) + assert json.loads(seen[0].content) == {"force": forced} + + +async def test_async_pause_accepts_a_204_from_an_older_api(monkeypatch, test_api_key): + _mock_api(monkeypatch, lambda: httpx.Response(204)) + + result = await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=False) + + +def test_pause_refusal_through_the_real_parsing(monkeypatch, test_api_key): + _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + json={ + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == REFUSED_MESSAGE + + +async def test_async_pause_refusal_through_the_real_parsing(monkeypatch, test_api_key): + _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + json={ + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == REFUSED_MESSAGE + + +class _Clock: + def __init__(self): + self.now = 0.0 + self.sleeps = [] + + def monotonic(self): + return self.now + + def sleep(self, delay): + self.sleeps.append(delay) + self.now += delay + + +class _StalledBody(httpx.SyncByteStream): + """A 503 body whose read eats 2 s of the clock and then times out.""" + + def __init__(self, clock): + self.clock = clock + + def __iter__(self): + self.clock.now += 2.0 + raise httpx.ReadTimeout("stalled") + yield b"" # pragma: no cover + + +def test_pause_whose_refusal_body_read_times_out_with_no_budget_left_reports_the_timeout( + monkeypatch, test_api_key +): + clock = _Clock() + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=_StalledBody(clock), + headers={"Content-Type": "application/json", "Content-Length": "40"}, + ), + clock=clock, + ) + + # A 1 s request timeout is the whole retry budget; the read outlives it. + with pytest.raises(Exception) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert isinstance(excinfo.value, (httpx.TimeoutException, TimeoutException)) + assert len(seen) == 1 + + +REFUSAL_CHUNKS = ( + b'{"code":503,"error_code":"sand', + b"box_pause_", + b'refused","message":"refused"}', +) + + +class _DripBody(httpx.SyncByteStream): + """A 503 body whose every chunk eats 0.6 s of the clock; ``tail`` chunks + follow the complete refusal and must never be pulled. Counts closes.""" + + def __init__(self, clock, chunks=REFUSAL_CHUNKS, tail=()): + self.clock = clock + self.chunks = chunks + self.tail = tail + self.closes = 0 + self.pulled = 0 + + def __iter__(self): + for chunk in self.chunks: + self.clock.now += 0.6 + self.pulled += 1 + yield chunk + for chunk in self.tail: + self.pulled += 1 + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncDripBody(httpx.AsyncByteStream): + def __init__(self, clock, chunks=REFUSAL_CHUNKS, tail=()): + self.clock = clock + self.chunks = chunks + self.tail = tail + self.closes = 0 + self.pulled = 0 + + async def __aiter__(self): + for chunk in self.chunks: + self.clock.now += 0.6 + self.pulled += 1 + yield chunk + for chunk in self.tail: + self.pulled += 1 + yield chunk + + async def aclose(self): + self.closes += 1 + + +def test_pause_whose_refusal_body_trickles_past_the_budget_still_surfaces_the_refusal( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + # Three chunks at 0.6 s against the 1 s request timeout: the peek is cut + # off on the third chunk, which completes the refusal, so the buffered + # bytes are used as the body and the live tail is never read. + with pytest.raises(SandboxPauseRefusedException): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +class _BigRefusalBody(httpx.SyncByteStream): + """A refusal body past the 8 KiB cap, in three chunks of 0.5 s each, so + only the cap stops the peek. Counts closes and pulls.""" + + BODY = ( + b'{"code":503,"error_code":"sandbox_pause_refused","message":"' + + b"x" * 9000 + + b'"}' + ) + + def __init__(self, clock): + self.clock = clock + self.closes = 0 + self.pulled = 0 + + def __iter__(self): + for chunk in (self.BODY[:4200], self.BODY[4200:8400], self.BODY[8400:]): + self.clock.now += 0.5 + self.pulled += 1 + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncBigRefusalBody(httpx.AsyncByteStream): + def __init__(self, clock): + self.clock = clock + self.closes = 0 + self.pulled = 0 + + async def __aiter__(self): + body = _BigRefusalBody.BODY + for chunk in (body[:4200], body[4200:8400], body[8400:]): + self.clock.now += 0.5 + self.pulled += 1 + yield chunk + + async def aclose(self): + self.closes += 1 + + +def test_pause_whose_refusal_body_passes_the_cap_with_no_budget_left_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _BigRefusalBody(clock) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=body, + headers={ + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + }, + ), + clock=clock, + ) + + # Over the cap on the second chunk at the 1 s request timeout: the 503 is + # handed on, but its third chunk would be waited for past the deadline, + # so the read ends there with a timeout instead of running on; the inner + # response is closed once. + with pytest.raises(httpx.ReadTimeout): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 2 + assert body.closes == 1 + + +# The refusal split so that the third chunk leaves the JSON open: the peek cut +# off there holds no refusal yet, and the body is handed on with a live tail. +OPEN_REFUSAL_CHUNKS = ( + b'{"code":503,"error_code":"sand', + b"box_pause_", + b'refused","message":"refused"', +) + + +def test_pause_whose_handed_on_body_trickles_past_the_deadline_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, chunks=OPEN_REFUSAL_CHUNKS, tail=(b"}",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + # Cut off on the third chunk with the JSON still open and no budget left: + # the 503 is handed on, but its tail would be read past the 1 s request + # timeout, so the read ends there with a timeout instead of running on. + with pytest.raises(httpx.ReadTimeout): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_handed_on_body_trickles_past_the_deadline_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, chunks=OPEN_REFUSAL_CHUNKS, tail=(b"}",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(httpx.ReadTimeout): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_refusal_body_passes_the_cap_with_no_budget_left_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncBigRefusalBody(clock) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=body, + headers={ + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + }, + ), + clock=clock, + ) + + with pytest.raises(httpx.ReadTimeout): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 2 + assert body.closes == 1 + + +async def test_async_pause_whose_refusal_body_trickles_past_the_budget_still_surfaces_the_refusal( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(SandboxPauseRefusedException): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +# A busy 503 (no refusal code) whose complete body lands in the very chunk that +# stops the peek: it must surface as the API error it is, not as a timeout. +BUSY_CHUNKS = (b'{"code":503,"message":"node is bu', b"sy persisting ", b'sandbox"}') + + +def test_pause_whose_busy_body_completes_in_the_cut_off_chunk_surfaces_the_busy_error( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, chunks=BUSY_CHUNKS, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(ServiceBusyException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "node is busy persisting sandbox" in str(excinfo.value) + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_busy_body_completes_in_the_cut_off_chunk_surfaces_the_busy_error( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, chunks=BUSY_CHUNKS, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(ServiceBusyException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "node is busy persisting sandbox" in str(excinfo.value) + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 diff --git a/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py b/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py index dd688cbf7b..a51feac2ae 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py @@ -11,7 +11,7 @@ def test_pause_filesystem_only(sandbox: Sandbox): boot_before = sandbox.files.read("/proc/sys/kernel/random/boot_id").strip() # Filesystem-only pause: only the rootfs is persisted, no memory snapshot. - assert sandbox.pause(mode="filesystem", request_timeout=120) + assert sandbox.pause(mode="filesystem", request_timeout=120).paused assert not sandbox.is_running() # Resuming a filesystem-only snapshot cold-boots (reboots) from the rootfs. diff --git a/packages/python-sdk/tests/test_api_exception.py b/packages/python-sdk/tests/test_api_exception.py index cd6dc81686..c32d81a1b2 100644 --- a/packages/python-sdk/tests/test_api_exception.py +++ b/packages/python-sdk/tests/test_api_exception.py @@ -3,6 +3,7 @@ from e2b.api import api_exception_from_code, handle_api_exception from e2b.exceptions import ( RateLimitException, + SandboxPauseRefusedException, ServiceBusyException, SandboxException, ) @@ -22,6 +23,43 @@ def test_refused_pause_is_a_busy_exception_with_the_status_and_message(): assert "node is busy persisting sandbox" in str(err) +def test_503_with_the_refused_code_is_a_pause_refused_exception_whatever_the_endpoint(): + reason = "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry" + err = handle_api_exception( + response( + 503, + ( + '{"code":503,"error_code":"sandbox_pause_refused","message":"' + + reason + + '"}' + ).encode(), + ) + ) + + assert isinstance(err, SandboxPauseRefusedException) + assert isinstance(err, ServiceBusyException) + assert err.status_code == 503 + assert err.error_code == "sandbox_pause_refused" + assert str(err) == reason + + +def test_503_with_another_code_stays_a_busy_exception(): + err = api_exception_from_code( + 503, "no capacity", error_code="sandbox_capacity_unavailable" + ) + + assert isinstance(err, ServiceBusyException) + assert not isinstance(err, SandboxPauseRefusedException) + + +def test_503_with_a_json_list_body_is_still_a_busy_exception(): + err = handle_api_exception(response(503, b'["not", "an", "object"]')) + + assert isinstance(err, ServiceBusyException) + assert not isinstance(err, SandboxPauseRefusedException) + assert err.status_code == 503 + + def test_503_without_a_body_is_still_a_busy_exception(): err = handle_api_exception(response(503)) diff --git a/packages/python-sdk/tests/test_rate_limit_retry_transport.py b/packages/python-sdk/tests/test_rate_limit_retry_transport.py index c4699a8b72..3316d5c05f 100644 --- a/packages/python-sdk/tests/test_rate_limit_retry_transport.py +++ b/packages/python-sdk/tests/test_rate_limit_retry_transport.py @@ -1,13 +1,21 @@ +import asyncio +import threading + import httpx import pytest from e2b.retry import ( + _AsyncResumedStream, + _Peek, + _ResumedStream, AsyncRetryableTransport, RetryableTransport, is_replayable, parse_retry_after, ) +JSON = {"Content-Type": "application/json"} + class FakeTransport(httpx.BaseTransport): def __init__(self, outcomes, retry_after="0"): @@ -26,7 +34,6 @@ def handle_request(self, request): outcome, headers={"Retry-After": self.retry_after} if self.retry_after else {}, content=b"response", - request=request, ) self.responses.append(response) return response @@ -52,7 +59,6 @@ async def handle_async_request(self, request): outcome, headers={"Retry-After": self.retry_after} if self.retry_after else {}, content=b"response", - request=request, ) self.responses.append(response) return response @@ -218,6 +224,1204 @@ def test_bad_gateway_is_retried_only_for_replayable_operations(method, path, att assert len(inner.requests) == attempts +class JsonTransport(httpx.BaseTransport): + """Answers each request with the next (status, body) pair.""" + + def __init__(self, outcomes): + self.outcomes = iter(outcomes) + self.requests = [] + + def handle_request(self, request): + self.requests.append(request) + status, body = next(self.outcomes) + return httpx.Response(status, json=body) + + +class JsonAsyncTransport(httpx.AsyncBaseTransport): + def __init__(self, outcomes): + self.outcomes = iter(outcomes) + self.requests = [] + + async def handle_async_request(self, request): + self.requests.append(request) + status, body = next(self.outcomes) + return httpx.Response(status, json=body) + + +REFUSED = {"code": 503, "error_code": "sandbox_pause_refused", "message": "refused"} +CAPACITY = {"code": 503, "error_code": "sandbox_capacity_unavailable", "message": "no"} + + +def test_service_unavailable_with_a_non_retryable_error_code_is_returned_after_one_request(): + inner = JsonTransport([(503, REFUSED)]) + sleeps = [] + + response = RetryableTransport( + inner, retries=3, sleep=sleeps.append, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert sleeps == [] + + +def test_service_unavailable_with_an_oversized_body_is_retried_without_reading_it(): + big = {"code": 503, "error_code": "sandbox_pause_refused", "message": "x" * 9000} + inner = JsonTransport([(503, big), (200, {})]) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +class _ChunkedStream(httpx.SyncByteStream): + """A 503 body of unknown length that records whether it was read.""" + + def __init__(self, chunks, raise_=None): + self.chunks = chunks + self.raise_ = raise_ + self.read = False + + def __iter__(self): + self.read = True + if self.raise_ is not None: + raise self.raise_ + yield from self.chunks + + +class _AsyncChunkedStream(httpx.AsyncByteStream): + def __init__(self, chunks, raise_=None): + self.chunks = chunks + self.raise_ = raise_ + self.read = False + + async def __aiter__(self): + self.read = True + if self.raise_ is not None: + raise self.raise_ + for chunk in self.chunks: + yield chunk + + +class StreamTransport(httpx.BaseTransport): + """Answers each request with the next prepared response factory.""" + + def __init__(self, factories): + self.factories = iter(factories) + self.requests = [] + + def handle_request(self, request): + self.requests.append(request) + return next(self.factories)(request) + + +class AsyncStreamTransport(httpx.AsyncBaseTransport): + def __init__(self, factories): + self.factories = iter(factories) + self.requests = [] + + async def handle_async_request(self, request): + self.requests.append(request) + return next(self.factories)(request) + + +def test_chunked_refusal_without_content_length_is_returned_after_one_request(): + body = _ChunkedStream([b'{"error_code":"sandbox_', b'pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + # The body read for the peek is still the caller's to read. + assert response.json()["error_code"] == "sandbox_pause_refused" + + +async def test_async_chunked_refusal_without_content_length_is_returned_after_one_request(): + body = _AsyncChunkedStream([b'{"error_code":"sandbox_', b'pause_refused"}']) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + assert response.json()["error_code"] == "sandbox_pause_refused" + + +def test_oversized_chunked_body_is_retried(): + body = _ChunkedStream( + [b'{"error_code":"sandbox_pause_refused","pad":"' + b"x" * 9000 + b'"}'] + ) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_non_json_503_body_is_retried_without_reading_it(): + body = _ChunkedStream([b"busy"]) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=body, headers={"Content-Type": "text/html"} + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is False + + +def test_content_type_is_matched_by_media_type_whatever_its_case_and_parameters(): + body = _ChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Type": "Application/JSON; charset=UTF-8"}, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + + +def test_service_unavailable_whose_body_read_fails_is_retried(): + body = _ChunkedStream([], raise_=httpx.ReadTimeout("stalled")) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "40", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is True + + +async def test_async_service_unavailable_whose_body_read_fails_is_retried(): + body = _AsyncChunkedStream([], raise_=httpx.ReadTimeout("stalled")) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "40", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is True + + +async def test_async_service_unavailable_with_a_refusal_body_of_known_length_is_returned(): + body = _AsyncChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "38", "Content-Type": "application/json"}, + ), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + + +class _SlowStream(httpx.SyncByteStream): + """A body whose read advances the fake clock, then yields or raises.""" + + def __init__(self, clock, seconds, chunks, raise_=None): + self.clock = clock + self.seconds = seconds + self.chunks = chunks + self.raise_ = raise_ + + def __iter__(self): + self.clock.now += self.seconds + if self.raise_ is not None: + raise self.raise_ + yield from self.chunks + + +class _AsyncSlowStream(httpx.AsyncByteStream): + def __init__(self, clock, seconds, chunks, raise_=None): + self.clock = clock + self.seconds = seconds + self.chunks = chunks + self.raise_ = raise_ + + async def __aiter__(self): + self.clock.now += self.seconds + if self.raise_ is not None: + raise self.raise_ + for chunk in self.chunks: + yield chunk + + +BUDGET_1S = {"timeout": {"connect": None, "read": 1.0, "write": None, "pool": None}} + + +def test_failed_503_body_read_with_no_budget_left_reraises_the_read_error(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 2.0, [], raise_=httpx.ReadTimeout("stalled")), + headers={"Content-Length": "40", **JSON}, + ), + ] + ) + + with pytest.raises(httpx.ReadTimeout, match="stalled"): + RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # The read outlived the 1 s budget: no retry, and the failed read is the + # truthful outcome (no fabricated empty body for the API client to parse). + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_failed_503_body_read_with_no_budget_left_reraises_the_read_error(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_AsyncSlowStream( + clock, 2.0, [], raise_=httpx.ReadTimeout("stalled") + ), + headers={"Content-Length": "40", **JSON}, + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + with pytest.raises(httpx.ReadTimeout, match="stalled"): + await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +def test_complete_503_body_read_with_no_budget_left_returns_it_with_the_body(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 2.0, [b'{"code":503,"message":"busy"}']), + headers={"Content-Length": "29", **JSON}, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + + +# A refusal in three chunks; the third completes the JSON. +DRIP = [b'{"error_code":"sand', b"box_pause_", b'refused"}'] +# The same refusal split so that the third chunk leaves the JSON open: a peek +# cut off there holds a prefix that does not parse, and hands the body on with +# a live tail. +OPEN_DRIP = [b'{"error_code":"sand', b"box_pause_", b'refused"', b"}"] + + +class _DripStream(httpx.SyncByteStream): + """A body whose every chunk advances the fake clock; counts closes.""" + + def __init__(self, clock, per_chunk, chunks): + self.clock = clock + self.per_chunk = per_chunk + self.chunks = chunks + self.closes = 0 + + def __iter__(self): + for chunk in self.chunks: + self.clock.now += self.per_chunk + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncDripStream(httpx.AsyncByteStream): + def __init__(self, clock, per_chunk, chunks): + self.clock = clock + self.per_chunk = per_chunk + self.chunks = chunks + self.closes = 0 + + async def __aiter__(self): + for chunk in self.chunks: + self.clock.now += self.per_chunk + yield chunk + + async def aclose(self): + self.closes += 1 + + +async def test_async_peek_stops_after_its_wall_clock_budget_and_the_503_is_retried(): + clock = FakeClock() + # The cut-off falls on the third chunk, which leaves the JSON open, so + # the buffered prefix is not yet a refusal. + chunks = OPEN_DRIP + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, chunks), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, + retries=3, + sleep=sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # 1.2 s into a 1 s peek budget the read stops with the JSON still open: + # not a refusal, retried. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_peek_stops_after_its_wall_clock_budget_and_the_503_is_retried(): + clock = FakeClock() + # The cut-off falls on the third chunk, which leaves the JSON open, so + # the buffered prefix is not yet a refusal. + chunks = OPEN_DRIP + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, chunks), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # 1.2 s into a 1 s peek budget the read stops with the JSON still open: + # not a refusal, retried. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_read_cut_off_by_the_budget_with_no_budget_left_hands_the_503_on_whole(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, DRIP), headers=JSON + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # Three chunks at 0.6 s against a 1 s budget: the third wait began past + # it and the request budget is gone too, so the 503 is the outcome and its + # body is handed on whole (peeked chunks, then the rest of the stream), as + # the JavaScript side does. + assert response.status_code == 503 + assert response.read() == b"".join(DRIP) + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_read_cut_off_by_the_budget_with_no_budget_left_hands_the_503_on_whole(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, DRIP), headers=JSON + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert (await response.aread()) == b"".join(DRIP) + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_complete_503_body_read_with_no_budget_left_returns_it_with_the_body(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_AsyncSlowStream(clock, 2.0, [b'{"code":503,"message":"busy"}']), + headers={"Content-Length": "29", **JSON}, + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + + +BIG_REFUSAL = ( + b'{"code":503,"error_code":"sandbox_pause_refused","message":"' + + b"x" * 9000 + + b'"}' +) +# The cap (8 KiB) is passed on the second chunk; the third is only ever read by +# the resumed stream's live-iterator replay. +BIG_CHUNKS = [BIG_REFUSAL[:4200], BIG_REFUSAL[4200:8400], BIG_REFUSAL[8400:]] +STREAMED_JSON = {**JSON, "Transfer-Encoding": "chunked"} + + +def test_over_cap_body_with_no_budget_left_is_handed_on_up_to_the_deadline(): + clock = FakeClock() + body = _DripStream(clock, 0.5, BIG_CHUNKS) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=STREAMED_JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # Past the byte cap on the second chunk (0.5 s per chunk keeps every wait + # within the 1 s peek budget, so only the cap stops the peek). The peek's + # time counts against the retry delay, so a budget exhausted after it + # means the clock is at the request deadline: the 503 is handed on, the + # stream-form rebuild carries neither a stale length nor the transfer + # encoding, the peeked chunks are replayed, and the tail is not waited + # for past the deadline. + assert response.status_code == 503 + assert "content-length" not in response.headers + assert "transfer-encoding" not in response.headers + chunks = response.iter_bytes() + assert [next(chunks), next(chunks)] == BIG_CHUNKS[:2] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + next(chunks) + assert body.closes == 1 + assert len(inner.requests) == 1 + + +async def test_async_over_cap_body_with_no_budget_left_is_handed_on_up_to_the_deadline(): + clock = FakeClock() + body = _AsyncDripStream(clock, 0.5, BIG_CHUNKS) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=STREAMED_JSON)] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert "content-length" not in response.headers + assert "transfer-encoding" not in response.headers + received = [] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + async for chunk in response.aiter_bytes(): + received.append(chunk) + assert received == BIG_CHUNKS[:2] + assert body.closes == 1 + assert len(inner.requests) == 1 + + +DEEP = b"[" * 3000 + + +def test_a_503_body_nested_too_deep_to_parse_is_not_a_refusal_and_is_retried(): + inner = StreamTransport( + [ + lambda request: httpx.Response(503, content=DEEP, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # json.loads raises RecursionError here, not ValueError; the peek must + # swallow it like any unparsable body. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_a_503_body_nested_too_deep_to_parse_is_not_a_refusal_and_is_retried(): + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, content=DEEP, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +class _GatedStream(httpx.SyncByteStream): + """Sync twin of ``_GatedAsyncStream`` on threading events.""" + + def __init__(self, chunks, gate_before, clock=None, per_chunk=0.0): + self.chunks = chunks + self.gate_before = gate_before + self.clock = clock + self.per_chunk = per_chunk + self.gate = threading.Event() + self.waiting = threading.Event() + self.closes = 0 + + def __iter__(self): + for index, chunk in enumerate(self.chunks): + if index == self.gate_before: + self.waiting.set() + self.gate.wait() + if self.clock is not None: + self.clock.now += self.per_chunk + yield chunk + + def close(self): + self.closes += 1 + + +def test_closing_the_resumed_response_while_a_reader_thread_iterates_closes_the_inner_once(): + # Built directly with the deadline ahead: through the transport a hand-on + # happens at the deadline, where the tail is never waited for. + body = _GatedStream(OPEN_DRIP, gate_before=3) + inner = httpx.Response(503, stream=body, headers=JSON) + rest = inner.iter_bytes() + peek = _Peek(buffered=[next(rest)], rest=rest) + request = httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause") + response = httpx.Response( + 503, + stream=_ResumedStream(inner, peek, request, float("inf"), lambda: 0.0), + request=request, + ) + + received = [] + reader = threading.Thread( + target=lambda: received.extend(response.iter_bytes()), daemon=True + ) + reader.start() + assert body.waiting.wait(5) + + # The reader is inside the live generator (awaiting the fourth chunk); + # closing it from here raises ValueError("generator already executing"), + # which close() tolerates while still closing the inner exactly once. + response.close() + assert body.closes == 1 + + body.gate.set() + reader.join(5) + assert not reader.is_alive() + assert b"".join(received) == b"".join(OPEN_DRIP) + assert body.closes == 1 + + +UNHASHABLE_CODE = { + "code": 503, + "error_code": ["sandbox_pause_refused"], + "message": "a proxy rewrote the body", +} + + +def test_a_503_whose_error_code_is_not_a_string_is_not_a_refusal_and_is_retried(): + inner = StreamTransport( + [ + lambda request: httpx.Response(503, json=UNHASHABLE_CODE), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # A list is unhashable: without the string guard the membership test + # raised TypeError out of the transport. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_a_503_whose_error_code_is_not_a_string_is_not_a_refusal_and_is_retried(): + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, json=UNHASHABLE_CODE), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_a_cut_off_peek_whose_buffered_chunks_form_the_refusal_returns_it_without_retrying(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, DRIP), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # The third chunk's wait began past the 1 s budget, but it was the last + # one: the buffered bytes are the whole refusal, so no retry is needed + # although budget remains. + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_a_cut_off_peek_whose_buffered_chunks_form_the_refusal_returns_it_without_retrying(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, DRIP), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, + retries=3, + sleep=sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +class _GatedAsyncStream(httpx.AsyncByteStream): + """Yields ``chunks`` advancing the clock per chunk, but waits on ``gate`` + before the chunk at ``gate_before``; counts closes.""" + + def __init__(self, chunks, gate_before, clock=None, per_chunk=0.0): + self.chunks = chunks + self.gate_before = gate_before + self.clock = clock + self.per_chunk = per_chunk + self.gate = asyncio.Event() + self.waiting = asyncio.Event() + self.closes = 0 + + async def __aiter__(self): + for index, chunk in enumerate(self.chunks): + if index == self.gate_before: + self.waiting.set() + await self.gate.wait() + if self.clock is not None: + self.clock.now += self.per_chunk + yield chunk + + async def aclose(self): + self.closes += 1 + + +async def test_async_cancellation_during_the_peek_closes_the_inner_response_once(): + body = _GatedAsyncStream( + [b'{"code":503,"error_code":"sand', b'box_pause_refused","message":"r"}'], + gate_before=1, + ) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + async def sleep(_): + return None + + task = asyncio.create_task( + AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request( + "POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}" + ) + ) + ) + await body.waiting.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + # The peek was awaiting the second chunk; the inner response is closed + # exactly once and the cancellation propagates. + assert body.closes == 1 + + +async def test_async_closing_the_resumed_response_while_a_reader_awaits_closes_the_inner_once(): + body = _GatedAsyncStream(OPEN_DRIP, gate_before=3) + inner = httpx.Response(503, stream=body, headers=JSON) + rest = inner.aiter_bytes() + peek = _Peek(buffered=[await rest.__anext__()], rest=rest) + request = httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause") + response = httpx.Response( + 503, + stream=_AsyncResumedStream(inner, peek, request, float("inf"), lambda: 0.0), + request=request, + ) + + received = [] + + async def read(): + async for chunk in response.aiter_bytes(): + received.append(chunk) + + reader = asyncio.create_task(read()) + await body.waiting.wait() + + # A second task closes while the reader awaits the fourth chunk inside the + # running generator: that generator cannot be closed from outside, and the + # inner response is still closed, exactly once. + await response.aclose() + assert body.closes == 1 + + body.gate.set() + await reader + assert b"".join(received) == b"".join(OPEN_DRIP) + assert body.closes == 1 + + +def test_handed_on_tail_is_bounded_by_the_request_deadline(): + clock = FakeClock() + body = _DripStream(clock, 0.6, OPEN_DRIP) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + assert response.status_code == 503 + + # The three peeked chunks are replayed; the fourth would be waited for + # past the deadline, so the read ends with a timeout there. + chunks = response.iter_bytes() + assert [next(chunks) for _ in range(3)] == OPEN_DRIP[:3] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + next(chunks) + assert body.closes == 1 + + +async def test_async_handed_on_tail_is_bounded_by_the_request_deadline(): + clock = FakeClock() + body = _AsyncDripStream(clock, 0.6, OPEN_DRIP) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + assert response.status_code == 503 + + received = [] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + async for chunk in response.aiter_bytes(): + received.append(chunk) + assert received == OPEN_DRIP[:3] + assert body.closes == 1 + + +BUSY_DRIP = [b'{"code":503,"message":"bu', b"sy", b'"}', b"\n"] + + +def test_a_complete_busy_body_in_the_cut_off_chunk_is_retried_without_reading_the_tail(): + clock = FakeClock() + body = _DripStream(clock, 0.6, BUSY_DRIP) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # The third chunk's wait began past the budget but completed the JSON: a + # plain busy 503 read in full, retried like any other with the tail left. + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.closes == 1 + + +def test_a_complete_busy_body_in_the_cut_off_chunk_with_no_budget_left_is_returned_whole(): + clock = FakeClock() + body = _DripStream(clock, 0.6, BUSY_DRIP) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + assert body.closes == 1 + + +def test_rebuilt_refusal_drops_transfer_headers_and_native_extensions(): + body = _ChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={**JSON, "Transfer-Encoding": "chunked", "X-Keep": "1"}, + extensions={ + "http_version": b"HTTP/2", + "get_trailers": lambda: {"bound": "to the closed response"}, + }, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert "transfer-encoding" not in response.headers + assert response.headers["x-keep"] == "1" + assert response.extensions == {"http_version": b"HTTP/2"} + assert response.request.url.path == "/sandboxes/sbx-1/pause" + + +def test_time_spent_reading_the_503_body_counts_towards_the_delay(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 0.02, [b'{"code":503}']), + headers={"Content-Length": "12", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # Backoff 0.05 s at attempt 0, minus the 0.02 s the read took. + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert clock.sleeps == [pytest.approx(0.03)] + + +def test_service_unavailable_with_another_error_code_is_retried(): + inner = JsonTransport([(503, CAPACITY), (200, {})]) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request(httpx.Request("POST", "https://api.test/sandboxes", content=b"p")) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_service_unavailable_with_a_non_retryable_error_code_is_returned_after_one_request(): + inner = JsonAsyncTransport([(503, REFUSED)]) + sleeps = [] + + async def sleep(delay): + sleeps.append(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert sleeps == [] + + def test_service_unavailable_is_retried_for_non_replayable_operation(): inner = FakeTransport([503, 200], retry_after=None) diff --git a/spec/openapi.yml b/spec/openapi.yml index f293777631..bec548a7a6 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -1157,6 +1157,24 @@ components: and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. + force: + type: boolean + default: false + description: >- + Take the pause even when the sandbox guest agent is unresponsive. + Applies to memory pauses only. The memory snapshot may not resume; + a reboot resume recovers the disk state. Defaults to false. + + SandboxPauseResult: + type: object + required: + - forced + properties: + forced: + type: boolean + description: >- + Whether the pause was taken with the force override, skipping the + guest agent health check. SandboxForkRequest: type: object @@ -2194,8 +2212,11 @@ components: description: >- Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, - sandbox_no_compatible_node, sandbox_create_failed, internal_server_error, - secret_limit_reached. + sandbox_no_compatible_node, sandbox_create_failed, sandbox_pause_refused + (the node refused the pause and the sandbox keeps running: the guest + agent did not answer the health check, or the node was busy; retry, or + pause with force when the guest agent is unresponsive), + internal_server_error, secret_limit_reached. message: type: string description: Error @@ -3307,8 +3328,14 @@ paths: schema: $ref: "#/components/schemas/SandboxPauseRequest" responses: - "204": + "200": description: The sandbox was paused successfully and can be resumed + content: + application/json: + schema: + $ref: "#/components/schemas/SandboxPauseResult" + "400": + $ref: "#/components/responses/400" "409": $ref: "#/components/responses/409" "404":