From 612f5ecb2b5b023cb125be7ca4f93e938274a589 Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 14 Oct 2026 23:38:26 +0000 Subject: [PATCH 1/3] feat(api-clients): pause takes force and answers 200 with the pause result What changed: spec/openapi.yml takes the pause contract from belt pull request 3536 (head b65468a1a7, oss/runtime/spec/openapi.yml): SandboxPauseRequest gains an optional boolean `force` (default false), a successful pause answers 200 with the SandboxPauseResult object `{"forced": boolean}` instead of 204, the endpoint documents 400 (force on a filesystem-only pause), and the sandbox_pause_refused clause is inserted into the existing Error.error_code list with belt's sentence verbatim; the other codes belt's spec has gained are not pulled in. The JavaScript schema and the Python client are regenerated from it with the repo's generators (redocly 2.54.3 + openapi-typescript, e2b-openapi-python-client 0.26.2 + ruff), at the pins in codegen.Dockerfile and the python-sdk codegen group. Why: the node may refuse a pause (503 with error_code sandbox_pause_refused) and keep the sandbox running, and lifts the guest-agent health check only when the caller passes force; the SDKs need the field, the result type and the code to expose that. Only the pause hunks are taken: belt's spec is ahead of the runtime pin in spec/runtime-ref by unrelated changes (egress proxy TLS, pauseMode, admin sandbox lookup, idempotency keys). spec/openapi.yml is normally copybara-synced from e2b-dev/runtime at that pin, so the generated-files check stays red until runtime-ref moves to a runtime commit that carries this contract; bump the pin and re-run `make codegen` then. Verification: the regenerated diff is limited to the pause endpoint, SandboxPauseRequest.force, the new SandboxPauseResult model and the Error.error_code clause; `pnpm build` (tsc --noEmit) and `ruff check` / `ruff format --check` / `ty check` pass. --- packages/js-sdk/src/api/schema.gen.ts | 18 +++++- .../post_sandboxes_sandbox_id_pause.py | 34 ++++++----- .../e2b/api/client/models/__init__.py | 2 + .../python-sdk/e2b/api/client/models/error.py | 2 + .../client/models/sandbox_pause_request.py | 11 ++++ .../api/client/models/sandbox_pause_result.py | 59 +++++++++++++++++++ spec/openapi.yml | 33 ++++++++++- 7 files changed, 139 insertions(+), 20 deletions(-) create mode 100644 packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py diff --git a/packages/js-sdk/src/api/schema.gen.ts b/packages/js-sdk/src/api/schema.gen.ts index cb39bcc138..952bc6ef3e 100644 --- a/packages/js-sdk/src/api/schema.gen.ts +++ b/packages/js-sdk/src/api/schema.gen.ts @@ -449,12 +449,15 @@ export interface paths { }; responses: { /** @description The sandbox was paused successfully and can be resumed */ - 204: { + 200: { headers: { [name: string]: unknown; }; - content?: never; + content: { + "application/json": components["schemas"]["SandboxPauseResult"]; + }; }; + 400: components["responses"]["400"]; 401: components["responses"]["401"]; 404: components["responses"]["404"]; 409: components["responses"]["409"]; @@ -2258,7 +2261,7 @@ export interface components { * @description Error code */ code: number; - /** @description Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, internal_server_error, secret_limit_reached. */ + /** @description Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, sandbox_pause_refused (the node refused the pause and the sandbox keeps running: the guest agent did not answer the health check, or the node was busy; retry, or pause with force when the guest agent is unresponsive), internal_server_error, secret_limit_reached. */ error_code?: string; /** @description Error */ message: string; @@ -2711,12 +2714,21 @@ export interface components { */ SandboxOnTimeout: "kill" | "pause"; SandboxPauseRequest: { + /** + * @description Take the pause even when the sandbox guest agent is unresponsive. Applies to memory pauses only. The memory snapshot may not resume; a reboot resume recovers the disk state. Defaults to false. + * @default false + */ + force?: boolean; /** * @description Whether to capture a full memory snapshot. When false, only the filesystem is persisted and resuming the sandbox cold-boots (reboots) it from disk, losing in-memory state, running processes, and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. * @default true */ memory?: boolean; }; + SandboxPauseResult: { + /** @description Whether the pause was taken with the force override, skipping the guest agent health check. */ + forced: boolean; + }; SandboxRefreshRequest: { /** @description Duration for which the sandbox should be kept alive in seconds */ duration?: number; diff --git a/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py b/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py index f0989b19e5..cfc65902ea 100644 --- a/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py +++ b/packages/python-sdk/e2b/api/client/api/sandboxes/post_sandboxes_sandbox_id_pause.py @@ -1,5 +1,5 @@ from http import HTTPStatus -from typing import Any, Optional, Union, cast +from typing import Any, Optional, Union import httpx @@ -7,6 +7,7 @@ from ...client import AuthenticatedClient, Client from ...models.error import Error from ...models.sandbox_pause_request import SandboxPauseRequest +from ...models.sandbox_pause_result import SandboxPauseResult from ...types import Response @@ -32,10 +33,15 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[Union[Any, Error]]: - if response.status_code == 204: - response_204 = cast(Any, None) - return response_204 +) -> Optional[Union[Error, SandboxPauseResult]]: + if response.status_code == 200: + response_200 = SandboxPauseResult.from_dict(response.json()) + + return response_200 + if response.status_code == 400: + response_400 = Error.from_dict(response.json()) + + return response_400 if response.status_code == 401: response_401 = Error.from_dict(response.json()) @@ -68,7 +74,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -82,7 +88,7 @@ def sync_detailed( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -96,7 +102,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[Union[Any, Error]] + Response[Union[Error, SandboxPauseResult]] """ kwargs = _get_kwargs( @@ -116,7 +122,7 @@ def sync( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Optional[Union[Any, Error]]: +) -> Optional[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -130,7 +136,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Union[Any, Error] + Union[Error, SandboxPauseResult] """ return sync_detailed( @@ -145,7 +151,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Response[Union[Any, Error]]: +) -> Response[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -159,7 +165,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[Union[Any, Error]] + Response[Union[Error, SandboxPauseResult]] """ kwargs = _get_kwargs( @@ -177,7 +183,7 @@ async def asyncio( *, client: AuthenticatedClient, body: SandboxPauseRequest, -) -> Optional[Union[Any, Error]]: +) -> Optional[Union[Error, SandboxPauseResult]]: """Pause sandbox Pause the sandbox @@ -191,7 +197,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Union[Any, Error] + Union[Error, SandboxPauseResult] """ return ( diff --git a/packages/python-sdk/e2b/api/client/models/__init__.py b/packages/python-sdk/e2b/api/client/models/__init__.py index 8aba151880..1d25d5eca2 100644 --- a/packages/python-sdk/e2b/api/client/models/__init__.py +++ b/packages/python-sdk/e2b/api/client/models/__init__.py @@ -50,6 +50,7 @@ from .sandbox_network_update_config_rules import SandboxNetworkUpdateConfigRules from .sandbox_on_timeout import SandboxOnTimeout from .sandbox_pause_request import SandboxPauseRequest +from .sandbox_pause_result import SandboxPauseResult from .sandbox_refresh_request import SandboxRefreshRequest from .sandbox_snapshot_request import SandboxSnapshotRequest from .sandbox_state import SandboxState @@ -132,6 +133,7 @@ "SandboxNetworkUpdateConfigRules", "SandboxOnTimeout", "SandboxPauseRequest", + "SandboxPauseResult", "SandboxRefreshRequest", "SandboxSnapshotRequest", "SandboxState", diff --git a/packages/python-sdk/e2b/api/client/models/error.py b/packages/python-sdk/e2b/api/client/models/error.py index a0a99778ac..7739540e8d 100644 --- a/packages/python-sdk/e2b/api/client/models/error.py +++ b/packages/python-sdk/e2b/api/client/models/error.py @@ -17,6 +17,8 @@ class Error: message (str): Error error_code (Union[Unset, str]): Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, sandbox_no_compatible_node, sandbox_create_failed, + sandbox_pause_refused (the node refused the pause and the sandbox keeps running: the guest agent did not answer + the health check, or the node was busy; retry, or pause with force when the guest agent is unresponsive), internal_server_error, secret_limit_reached. """ diff --git a/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py b/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py index 0f36b2eb83..06c4164e6d 100644 --- a/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py +++ b/packages/python-sdk/e2b/api/client/models/sandbox_pause_request.py @@ -17,19 +17,27 @@ class SandboxPauseRequest: persisted and resuming the sandbox cold-boots (reboots) it from disk, losing in-memory state, running processes, and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. Default: True. + force (Union[Unset, bool]): Take the pause even when the sandbox guest agent is unresponsive. Applies to memory + pauses only. The memory snapshot may not resume; a reboot resume recovers the disk state. Defaults to false. + Default: False. """ memory: Union[Unset, bool] = True + force: Union[Unset, bool] = False additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: memory = self.memory + force = self.force + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) if memory is not UNSET: field_dict["memory"] = memory + if force is not UNSET: + field_dict["force"] = force return field_dict @@ -38,8 +46,11 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: d = dict(src_dict) memory = d.pop("memory", UNSET) + force = d.pop("force", UNSET) + sandbox_pause_request = cls( memory=memory, + force=force, ) sandbox_pause_request.additional_properties = d diff --git a/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py b/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py new file mode 100644 index 0000000000..cd26ee7d33 --- /dev/null +++ b/packages/python-sdk/e2b/api/client/models/sandbox_pause_result.py @@ -0,0 +1,59 @@ +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="SandboxPauseResult") + + +@_attrs_define +class SandboxPauseResult: + """ + Attributes: + forced (bool): Whether the pause was taken with the force override, skipping the guest agent health check. + """ + + forced: bool + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + forced = self.forced + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "forced": forced, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + forced = d.pop("forced") + + sandbox_pause_result = cls( + forced=forced, + ) + + sandbox_pause_result.additional_properties = d + return sandbox_pause_result + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/spec/openapi.yml b/spec/openapi.yml index f293777631..bec548a7a6 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -1157,6 +1157,24 @@ components: and open connections. Resume it with an explicit request (connect or resume); auto-resume, which can be triggered by arbitrary traffic, refuses such a sandbox. Defaults to true. + force: + type: boolean + default: false + description: >- + Take the pause even when the sandbox guest agent is unresponsive. + Applies to memory pauses only. The memory snapshot may not resume; + a reboot resume recovers the disk state. Defaults to false. + + SandboxPauseResult: + type: object + required: + - forced + properties: + forced: + type: boolean + description: >- + Whether the pause was taken with the force override, skipping the + guest agent health check. SandboxForkRequest: type: object @@ -2194,8 +2212,11 @@ components: description: >- Machine-readable semantic error code. Not a closed set; initial values: sandbox_capacity_unavailable, sandbox_placement_timeout, - sandbox_no_compatible_node, sandbox_create_failed, internal_server_error, - secret_limit_reached. + sandbox_no_compatible_node, sandbox_create_failed, sandbox_pause_refused + (the node refused the pause and the sandbox keeps running: the guest + agent did not answer the health check, or the node was busy; retry, or + pause with force when the guest agent is unresponsive), + internal_server_error, secret_limit_reached. message: type: string description: Error @@ -3307,8 +3328,14 @@ paths: schema: $ref: "#/components/schemas/SandboxPauseRequest" responses: - "204": + "200": description: The sandbox was paused successfully and can be resumed + content: + application/json: + schema: + $ref: "#/components/schemas/SandboxPauseResult" + "400": + $ref: "#/components/responses/400" "409": $ref: "#/components/responses/409" "404": From 455640316941c028a56047644c3ccacd25df728a Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 14 Oct 2026 23:38:26 +0000 Subject: [PATCH 2/3] feat(js-sdk)!: pause takes force and returns a SandboxPauseResult; SandboxPauseRefusedError for a refused pause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What changed: `SandboxPauseOpts.force` is sent in the pause request body when set (omitted otherwise, like `memory`). `SandboxApi.pause()`, `Sandbox.pause()` and the deprecated `betaPause()` return `Promise` (an interface `{ paused, forced }`, exported) instead of a boolean: 200 answers `{ paused: true, forced: body.forced }`, 409 (already paused) `{ paused: false, forced: false }`, and a 204 from an older API `{ paused: true, forced: false }`. The CLI's `sandbox pause` reads `paused` from the result, takes `-f, --force` (passed only when given, so the SDK omits it from the body otherwise), and says when the result reports a forced pause. A 503 whose body carries `error_code: "sandbox_pause_refused"` is raised as the new `SandboxPauseRefusedError` (exported; `statusCode` 503, `errorCode`, subclass of `ServiceBusyError` so existing handlers keep working). The mapping lives in `apiErrorFromCode` / `handleApiError`, keyed on the body's `error_code` and passed through for errors embedded in fork results too, so any endpoint answering the code gets the typed error; the pause call site keeps only the 409 → `paused: false` case. The message is the API's verbatim: belt's messages already name the reason and what to do for it (retry, retry without force, or pause with force and the reboot caveat), so the SDK appends nothing and the general guidance lives in the class docs. A 503 without the code stays a `ServiceBusyError`, 404 stays `SandboxNotFoundError`, and 400 (force on a filesystem-only pause) is the generic `SandboxError` carrying the status. The code string is one constant, `SANDBOX_PAUSE_REFUSED_ERROR_CODE`, exported from the package and read by the error class, the mapping and the retry predicate. The transport no longer replays a 503 carrying that code: `withRetry` peeks at a 503 body and returns the response when its `error_code` is non-retryable, so a refusal surfaces after one request instead of re-running the node's health probe and restore cycle on each of the retries. The peek runs only for a body whose media type is JSON (lower-cased, parameters dropped), unconsumed and unlocked, on a request not already aborted (anything else is "not a refusal" and retried as before, instead of a synchronous TypeError failing the request); it reads a clone through a reader it owns, capped at 8 KiB and bounded by its own inspection budget of max(retry delay, 1 s) within the remaining request budget — so a slow refusal body is still caught when the first backoff is 50 ms, and a trickling non-refusal body costs at most a second per attempt — and by the request signal, waited through the injected sleep so RetryDependencies.sleep governs all waiting (clamped so a huge budget cannot overflow setTimeout); when it stops early the reader is cancelled so the abandoned read stops consuming the stream, the time spent counts towards the delay, the deadline is re-checked, and the 503 is retried as before, so a stalled or trickling body cannot eat the retry budget. The caller's body stays untouched. Other 503s are retried as before. The CLI prints a refused pause's API message (no stack) before exiting 1. The docs on the option, the methods and the error class describe the refusal as the node refusing the pause with the sandbox kept running and the reason in the message (belt emits the code for an unresponsive guest agent, a busy node and a forced pause it cannot pin), and present force as the answer for the unresponsive-guest reason only, with the reboot caveat. Why: the API now refuses a memory pause when the sandbox guest agent is unresponsive and keeps the sandbox running; callers need to tell that apart from a busy node (retry) and need the override to snapshot anyway, accepting that the memory may not resume (a resume with `onResume: 'reboot'` recovers the disk state). The result object is a breaking change, decided by the operator: `forced` is informative, not an echo of the option, since the API reports `forced: false` when a forced request joined a normal pause already in flight, and a boolean cannot carry it. Migration: read `paused` where the boolean was read. Verification: unit tests (msw-mocked API) cover force omitted / true / false / with mode in the body; 200 with forced false and true; forced reported false for a forced request that joined a normal pause; 204 compatibility; 409; the instance method; the coded refusal mapped to SandboxPauseRefusedError (still instanceof ServiceBusyError) with the API message verbatim, after exactly one request; the busy-node refusal; a 503 with no error_code staying ServiceBusyError; 400 carrying the status; 404. handleApiError tests cover the central mapping on the code regardless of endpoint and a 503 with another code staying ServiceBusyError. The transport tests assert a coded 503 is returned after one request with its body intact, a 503 with another code is still retried, a stalled body is retried after each attempt's inspection budget and delay (sleeps 1000/50, 1000/100, 1000/200 ms on the stub clock, four requests), a refusal arriving within the delay and one arriving after the first backoff but within the inspection budget both short-circuit, a mixed-case Content-Type is matched by media type, a dripping body (cancel-safe source) is retried with the peek's reader cancelled (the retried sources cancelled, no further pulls past the tee's high-water mark; the untouched-body assertions sit on the peeked-and-returned refusal), an oversized body is a plain 503, and a consumed body, a locked body and an already-aborted request are not peeked at. The whole js-sdk unit project runs with zero unhandled errors; its only failures without an API key are the live-sandbox suites. CLI tests cover the default, --force / -f, a forced request the API did not force, a refused pause printing the API message and exiting 1, and an already paused sandbox. `pnpm build`, oxlint, prettier, the CLI typecheck and lint, and the unit suites pass (249 js-sdk tests across the five touched files, 5 CLI tests). --- packages/cli/src/commands/sandbox/pause.ts | 37 +- .../cli/tests/commands/sandbox/pause.test.ts | 102 ++++++ packages/js-sdk/src/api/index.ts | 18 +- packages/js-sdk/src/errors.ts | 35 ++ packages/js-sdk/src/index.ts | 3 + packages/js-sdk/src/retry.ts | 139 ++++++- packages/js-sdk/src/sandbox/index.ts | 27 +- packages/js-sdk/src/sandbox/sandboxApi.ts | 68 +++- .../js-sdk/tests/api/handleApiError.test.ts | 30 ++ packages/js-sdk/tests/retry.test.ts | 340 ++++++++++++++++++ .../js-sdk/tests/sandbox/apiDefaults.test.ts | 29 +- packages/js-sdk/tests/sandbox/pause.test.ts | 180 ++++++++++ 12 files changed, 981 insertions(+), 27 deletions(-) create mode 100644 packages/cli/tests/commands/sandbox/pause.test.ts create mode 100644 packages/js-sdk/tests/sandbox/pause.test.ts diff --git a/packages/cli/src/commands/sandbox/pause.ts b/packages/cli/src/commands/sandbox/pause.ts index e11a11a5c7..ac2c116005 100644 --- a/packages/cli/src/commands/sandbox/pause.ts +++ b/packages/cli/src/commands/sandbox/pause.ts @@ -3,21 +3,38 @@ import * as commander from 'commander' import { ensureAPIKey } from 'src/api' import { asBold } from 'src/utils/format' import * as e2b from 'e2b' -import { NotFoundError } from 'e2b' +import { NotFoundError, ServiceBusyError } from 'e2b' -async function pauseSandbox(sandboxID: string, apiKey: string) { +async function pauseSandbox( + sandboxID: string, + apiKey: string, + force: boolean | undefined +) { try { - const paused = await e2b.Sandbox.pause(sandboxID, { apiKey }) - if (paused) { - console.log(`Sandbox ${asBold(sandboxID)} has been paused`) - } else { + // Left out unless given, so the SDK omits it from the request body. + const { paused, forced } = await e2b.Sandbox.pause(sandboxID, { + apiKey, + ...(force ? { force: true } : {}), + }) + if (!paused) { console.log(`Sandbox ${asBold(sandboxID)} is already paused`) + } else if (forced) { + console.log( + `Sandbox ${asBold(sandboxID)} has been paused with force: the snapshot was taken without the guest agent health check, so its memory may not resume (a reboot resume recovers the disk state)` + ) + } else { + console.log(`Sandbox ${asBold(sandboxID)} has been paused`) } } catch (err: unknown) { if (err instanceof NotFoundError) { console.error(`Sandbox ${asBold(sandboxID)} wasn't found`) process.exit(1) } + if (err instanceof ServiceBusyError) { + // The API's message says what to do: retry, or pause with --force. + console.error(err.message) + process.exit(1) + } throw err } } @@ -29,10 +46,14 @@ export const pauseCommand = new commander.Command('pause') `pause the sandbox specified by ${asBold('')}` ) .alias('ps') - .action(async (sandboxID: string) => { + .option( + '-f, --force', + 'pause even when the sandbox guest agent is unresponsive: the snapshot is taken without the health check and its memory may not resume (a reboot resume recovers the disk state)' + ) + .action(async (sandboxID: string, { force }: { force?: boolean }) => { try { const apiKey = ensureAPIKey() - await pauseSandbox(sandboxID, apiKey) + await pauseSandbox(sandboxID, apiKey, force) } catch (err: unknown) { console.error(err) process.exit(1) diff --git a/packages/cli/tests/commands/sandbox/pause.test.ts b/packages/cli/tests/commands/sandbox/pause.test.ts new file mode 100644 index 0000000000..8855b7f324 --- /dev/null +++ b/packages/cli/tests/commands/sandbox/pause.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + pause: vi.fn(), + ensureAPIKey: vi.fn(() => 'test-api-key'), +})) + +class ServiceBusyError extends Error {} +class SandboxPauseRefusedError extends ServiceBusyError {} + +vi.mock('e2b', () => ({ + Sandbox: { + pause: mocks.pause, + }, + NotFoundError: class NotFoundError extends Error {}, + ServiceBusyError, +})) + +vi.mock('../../../src/api', () => ({ + ensureAPIKey: mocks.ensureAPIKey, +})) + +describe('sandbox pause', () => { + let logSpy: ReturnType + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + async function run(...args: string[]) { + const { pauseCommand } = await import('../../../src/commands/sandbox/pause') + await pauseCommand.parseAsync(['node', 'e2b', ...args]) + } + + test('pauses without force by default, leaving force out of the options', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: false }) + + await run('sbx-1') + + expect(mocks.pause).toHaveBeenCalledWith('sbx-1', { + apiKey: 'test-api-key', + }) + expect(mocks.pause.mock.calls[0][1]).not.toHaveProperty('force') + expect(logSpy.mock.calls.flat().join('\n')).toContain('has been paused') + expect(logSpy.mock.calls.flat().join('\n')).not.toContain('with force') + }) + + test('--force passes force and reports a forced pause', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: true }) + + await run('sbx-1', '--force') + + expect(mocks.pause).toHaveBeenCalledWith('sbx-1', { + apiKey: 'test-api-key', + force: true, + }) + expect(logSpy.mock.calls.flat().join('\n')).toContain( + 'has been paused with force' + ) + }) + + test('a forced request the API did not force is reported as a plain pause', async () => { + mocks.pause.mockResolvedValue({ paused: true, forced: false }) + + await run('sbx-1', '-f') + + expect(logSpy.mock.calls.flat().join('\n')).not.toContain('with force') + }) + + test('a refused pause prints the API message, not a stack, and exits 1', async () => { + const message = + "Sandbox 'sbx-1' cannot be paused right now because the sandbox guest agent is not responding: retry, or pause with force" + mocks.pause.mockRejectedValue(new SandboxPauseRefusedError(message)) + const errorSpy = vi + .spyOn(console, 'error') + .mockImplementation(() => undefined) + const exitSpy = vi + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as never) + + await run('sbx-1') + + // The first thing printed is the API message itself; with process.exit + // mocked the command carries on past it, so only the first call counts. + expect(errorSpy.mock.calls[0]).toEqual([message]) + expect(exitSpy).toHaveBeenNthCalledWith(1, 1) + }) + + test('an already paused sandbox is reported as such', async () => { + mocks.pause.mockResolvedValue({ paused: false, forced: false }) + + await run('sbx-1') + + expect(logSpy.mock.calls.flat().join('\n')).toContain('is already paused') + }) +}) diff --git a/packages/js-sdk/src/api/index.ts b/packages/js-sdk/src/api/index.ts index 23d9a06307..c4118c7930 100644 --- a/packages/js-sdk/src/api/index.ts +++ b/packages/js-sdk/src/api/index.ts @@ -7,6 +7,8 @@ import { ConnectionConfig } from '../connectionConfig' import { AuthenticationError, RateLimitError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, + SandboxPauseRefusedError, ServiceBusyError, SandboxError, } from '../errors' @@ -16,7 +18,9 @@ import { withRetry } from '../retry' /** * Map an API error code and message to the matching error class — the same * mapping {@link handleApiError} applies to HTTP responses, usable for error - * objects embedded in response bodies (e.g. per-fork results). + * objects embedded in response bodies (e.g. per-fork results). `errorCode` is + * the body's semantic `error_code`, which refines the status for the codes + * the SDK types (a refused pause). */ export function apiErrorFromCode( code: number, @@ -25,7 +29,8 @@ export function apiErrorFromCode( message: string, stackTrace?: string ) => Error = SandboxError, - stackTrace?: string + stackTrace?: string, + errorCode?: string ): Error { if (code === 401) { const message = 'Unauthorized, please check your credentials.' @@ -40,6 +45,12 @@ export function apiErrorFromCode( } if (code === 503) { + if (errorCode === SANDBOX_PAUSE_REFUSED_ERROR_CODE) { + // The API's message already says what to do for its reason. + return new SandboxPauseRefusedError( + content ? `${content}` : 'The node refused the pause, please retry' + ) + } const message = 'Service temporarily unavailable, please retry' return new ServiceBusyError(content ? `${message} - ${content}` : message) } @@ -72,7 +83,8 @@ export function handleApiError( status, response.error?.message ?? response.error, errorClass, - stackTrace + stackTrace, + response.error?.error_code ) } diff --git a/packages/js-sdk/src/errors.ts b/packages/js-sdk/src/errors.ts index 176fed680f..5383a07e2a 100644 --- a/packages/js-sdk/src/errors.ts +++ b/packages/js-sdk/src/errors.ts @@ -261,6 +261,41 @@ export class ServiceBusyError extends Error { } } +/** + * `error_code` of the 503 the API answers a pause with when the node refused + * it. Any endpoint answering this code is mapped to + * {@link SandboxPauseRefusedError}. + */ +export const SANDBOX_PAUSE_REFUSED_ERROR_CODE = 'sandbox_pause_refused' + +/** + * Thrown when the node refused a pause (HTTP 503 with + * `error_code: "sandbox_pause_refused"`). The sandbox keeps running with its + * state intact; the message is the API's, and names the reason and what to + * do about it. In general: retry, and when the reason is an unresponsive + * sandbox guest agent, pause with `force: true` to take the snapshot without + * the health check — its memory may then not resume, and a resume with + * `onResume: 'reboot'` recovers the disk state. + * + * Unlike other 503s, a refusal carrying this code is not replayed by the + * SDK's transport retries: it is a decision, not transient capacity, so it + * surfaces after a single request and the caller decides what to do. + * + * Subclass of {@link ServiceBusyError}, which a refused pause surfaced as + * before, so existing handlers keep working. + */ +export class SandboxPauseRefusedError extends ServiceBusyError { + /** + * Semantic error code of the API response: always `sandbox_pause_refused`. + */ + readonly errorCode = SANDBOX_PAUSE_REFUSED_ERROR_CODE + + constructor(message: string) { + super(message) + this.name = 'SandboxPauseRefusedError' + } +} + /** * Thrown when the build fails. */ diff --git a/packages/js-sdk/src/index.ts b/packages/js-sdk/src/index.ts index 55a74e6af3..07ada50f4f 100644 --- a/packages/js-sdk/src/index.ts +++ b/packages/js-sdk/src/index.ts @@ -24,6 +24,8 @@ export { TimeoutError, RateLimitError, ServiceBusyError, + SandboxPauseRefusedError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, BuildError, FileUploadError, VolumeError, @@ -68,6 +70,7 @@ export type { SandboxForkOpts, SandboxMetricsOpts, SandboxPauseOpts, + SandboxPauseResult, SandboxState, SandboxListOpts, SandboxListOrder, diff --git a/packages/js-sdk/src/retry.ts b/packages/js-sdk/src/retry.ts index 903edce8bb..b6961de807 100644 --- a/packages/js-sdk/src/retry.ts +++ b/packages/js-sdk/src/retry.ts @@ -1,4 +1,7 @@ -import { InvalidArgumentError } from './errors' +import { + InvalidArgumentError, + SANDBOX_PAUSE_REFUSED_ERROR_CODE, +} from './errors' import { isReadableStreamLike } from './is' const MAX_RETRY_AFTER_SECONDS = 2_147_483 @@ -7,6 +10,9 @@ const BACKOFF_BASE_MS = 100 const BACKOFF_MAX_MS = 10_000 const BACKOFF_JITTER_MIN = 0.5 const RETRYABLE_STATUSES = new Set([429, 502, 503]) +// A 503 whose body carries one of these `error_code`s is a decision by the +// server, not transient capacity, and is answered after a single request. +const NON_RETRYABLE_ERROR_CODES = new Set([SANDBOX_PAUSE_REFUSED_ERROR_CODE]) // Error codes raised while establishing the connection, before any request // bytes are written: Node/undici (`ECONNREFUSED`, `ENOTFOUND`, ...), Bun // (`ConnectionRefused`). Post-write failures such as `ECONNRESET` are @@ -160,6 +166,113 @@ function backoffMs(attempt: number, random: () => number): number { ) } +const BODY_READ_TIMED_OUT = Symbol('body read timed out') + +/** The media type of a `Content-Type` value: lower-cased, parameters dropped. */ +function mediaType(contentType: string | null): string { + return (contentType ?? '').split(';', 1)[0].trim().toLowerCase() +} +// An error body larger than this cannot carry a code worth parsing; the read +// stops there and the 503 is retried as a plain one. +const MAX_ERROR_BODY_BYTES = 8 * 1024 +// Time a 503 body is peeked at for a non-retryable code, independent of the +// jittered backoff: a slow refusal body within this still short-circuits, and +// a trickling non-refusal body costs at most this per attempt. +const PEEK_BUDGET_MS = 1_000 + +/** + * Reads `reader` to the end into one buffer, or returns `undefined` once more + * than `limit` bytes arrived. + */ +async function readUpTo( + reader: ReadableStreamDefaultReader, + limit: number +): Promise { + const chunks: Uint8Array[] = [] + let size = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) break + size += value.byteLength + if (size > limit) return undefined + chunks.push(value) + } + const out = new Uint8Array(size) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.byteLength + } + return out +} + +/** + * Whether a 503 carries a non-retryable `error_code` (see + * `NON_RETRYABLE_ERROR_CODES`). Reads a clone through a reader this function + * owns, capped at `MAX_ERROR_BODY_BYTES` and bounded by `budgetMs` (the + * inspection budget, which overlaps the retry wait instead of adding to it; + * waited through `sleep` so the injected clock governs it) and the request + * signal; + * when it stops early the reader is cancelled so the abandoned read does not + * keep consuming the stream, and the answer is "not a refusal" so the 503 is + * retried as before. A body that cannot be peeked at (none, not JSON, + * consumed, locked) or a request already aborted is not a refusal either. The original body + * stays untouched for the caller. + */ +async function isNonRetryableRefusal( + response: Response, + budgetMs: number, + signal: AbortSignal, + sleep: (delayMs: number, signal: AbortSignal) => Promise +): Promise { + if (response.status !== 503 || signal.aborted) return false + // Only a JSON body can carry the code; a proxy's HTML page is not peeked at. + if (mediaType(response.headers.get('Content-Type')) !== 'application/json') { + return false + } + const { body } = response + if (!body || response.bodyUsed || body.locked) return false + let reader: ReadableStreamDefaultReader | undefined + const timer = new AbortController() + const onAbort = () => timer.abort(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + // Clamped so a huge budget cannot overflow setTimeout into a 1 ms timer. + const bounded = Math.min( + Math.max(0, budgetMs), + MAX_RETRY_AFTER_SECONDS * 1000 + ) + try { + const cloned = response.clone().body + if (!cloned) return false + reader = cloned.getReader() + const bytes = await Promise.race([ + readUpTo(reader, MAX_ERROR_BODY_BYTES).catch(() => undefined), + sleep(bounded, timer.signal).then( + () => BODY_READ_TIMED_OUT, + () => BODY_READ_TIMED_OUT + ), + ]) + if (typeof bytes === 'symbol' || bytes === undefined) { + // Settles the pending read with done, so readUpTo ends too. + void reader.cancel().catch(() => {}) + return false + } + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as { + error_code?: unknown + } + return ( + typeof parsed?.error_code === 'string' && + NON_RETRYABLE_ERROR_CODES.has(parsed.error_code) + ) + } catch { + void reader?.cancel().catch(() => {}) + return false + } finally { + timer.abort() + signal.removeEventListener('abort', onAbort) + } +} + /** * Delay before the next attempt, or `undefined` when the response is not * retried: `Retry-After` when the server sends a usable one, otherwise @@ -187,7 +300,10 @@ function retryDelayMs( * Retry requests after a 429 carrying `Retry-After`, a 503 or — for replayable * requests ({@link isReplayable}) — a 502 (using `Retry-After` when present, * exponential backoff otherwise), or a network failure (exponential backoff; - * see {@link isRetryableFetchError}). + * see {@link isRetryableFetchError}). A 503 whose body carries a + * non-retryable `error_code` (a refused pause) is returned after one request; + * the body is peeked at for an inspection budget of max(retry delay, 1 s) + * within the remaining request budget. */ export function withRetry( fetchImpl: typeof fetch, @@ -246,10 +362,27 @@ export function withRetry( return response } + // Peek at a 503 body for a non-retryable code for its own inspection + // budget (at least the retry delay, within the remaining request + // budget); the time spent peeking counts towards the delay. + const peekStart = monotonic() + if ( + await isNonRetryableRefusal( + response, + Math.min(Math.max(delayMs, PEEK_BUDGET_MS), deadline - peekStart), + request.signal, + sleep + ) + ) { + return response + } + const remainingDelayMs = Math.max(0, delayMs - (monotonic() - peekStart)) + if (monotonic() + remainingDelayMs >= deadline) return response + // Not awaited: some fetch interceptors never settle `cancel()`, and the // retry does not depend on the discarded body having been released. void response.body?.cancel().catch(() => {}) - await sleep(delayMs, request.signal) + await sleep(remainingDelayMs, request.signal) } }) as typeof fetch } diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts index 3af8e5e3d4..2f2fe2bccb 100644 --- a/packages/js-sdk/src/sandbox/index.ts +++ b/packages/js-sdk/src/sandbox/index.ts @@ -28,6 +28,7 @@ import { SnapshotPaginator, CreateSnapshotOpts, SandboxPauseOpts, + SandboxPauseResult, } from './sandboxApi' import { getSignature } from './signature' import { compareVersions } from 'compare-versions' @@ -639,31 +640,47 @@ export class Sandbox extends SandboxApi { /** * Pause a sandbox by its ID. * - * @param opts connection options, plus `mode` to control the snapshot kind. + * The node may refuse the pause ({@link SandboxPauseRefusedError}): the + * sandbox keeps running and the message carries the reason. When the reason + * is an unresponsive sandbox guest agent, `force: true` takes the snapshot + * without the health check, in which case the memory snapshot may not + * resume and a resume with `onResume: 'reboot'` recovers the disk state. + * + * @param opts connection options, plus `mode` to control the snapshot kind + * and `force` to skip the guest agent health check. * With `mode: 'filesystem'`, the in-memory state is dropped and only the * filesystem is persisted; resuming such a sandbox cold-boots (reboots) it * from disk, losing running processes and open connections. Defaults to * `'full'` (memory and filesystem). * - * @returns `true` if the sandbox got paused, `false` if the sandbox was already paused. + * @returns the {@link SandboxPauseResult}: `paused` is `true` if this call + * paused the sandbox and `false` if it was already paused; `forced` whether + * the snapshot was taken without the guest agent health check (reported by + * the API, so a forced request that joined a normal pause already in + * flight comes back with `forced: false`). + * + * @throws {@link SandboxPauseRefusedError} if the node refused the pause; the message carries the reason. * * @example * ```ts * const sandbox = await Sandbox.create() - * await sandbox.pause() + * const { paused } = await sandbox.pause() * * // filesystem-only snapshot (resume reboots the sandbox) * await sandbox.pause({ mode: 'filesystem' }) + * + * // snapshot an unresponsive sandbox anyway; its memory may not resume + * const { forced } = await sandbox.pause({ force: true }) * ``` */ - async pause(opts?: SandboxPauseOpts): Promise { + async pause(opts?: SandboxPauseOpts): Promise { return await SandboxApi.pause(this.sandboxId, this.resolveApiOpts(opts)) } /** * @deprecated Use {@link Sandbox.pause} instead. */ - async betaPause(opts?: SandboxPauseOpts): Promise { + async betaPause(opts?: SandboxPauseOpts): Promise { return await SandboxApi.betaPause(this.sandboxId, this.resolveApiOpts(opts)) } diff --git a/packages/js-sdk/src/sandbox/sandboxApi.ts b/packages/js-sdk/src/sandbox/sandboxApi.ts index 50872b4881..2f6b44170f 100644 --- a/packages/js-sdk/src/sandbox/sandboxApi.ts +++ b/packages/js-sdk/src/sandbox/sandboxApi.ts @@ -585,6 +585,42 @@ export interface SandboxPauseOpts extends SandboxApiOpts { * `mode: 'filesystem'`, `keepMemory: true` is `mode: 'full'`. */ keepMemory?: boolean + + /** + * Take the pause even when the sandbox guest agent is unresponsive. + * + * A memory pause first checks that the guest agent answers. When it does + * not, the node refuses the pause ({@link SandboxPauseRefusedError}, whose + * message carries the reason) and the sandbox keeps running. With + * `force: true` the snapshot is taken without that check. Its memory may + * then not resume: if restoring it fails, resume with `onResume: 'reboot'` + * to recover the disk state. A refusal for another reason is not lifted + * by `force`. + * + * Applies to memory pauses only; the API rejects it together with + * `mode: 'filesystem'`. + * + * @default false + */ + force?: boolean +} + +/** + * Result of a pause. + */ +export interface SandboxPauseResult { + /** + * `true` if this call paused the sandbox, `false` if it was already paused. + */ + paused: boolean + + /** + * Whether the snapshot was taken with the `force` override, without the + * guest agent health check. Reports what happened rather than echoing the + * option: `false` when the sandbox was already paused, and also when a + * forced request joined a normal pause that was already in flight. + */ + forced: boolean } /** @@ -1588,15 +1624,25 @@ export class SandboxApi extends ClientFactory { /** * Pause the sandbox specified by sandbox ID. * + * The node may refuse the pause ({@link SandboxPauseRefusedError}): the + * sandbox keeps running and the message carries the reason. When the reason + * is an unresponsive sandbox guest agent, `force: true` takes the snapshot + * without the health check, in which case the memory snapshot may not + * resume and a resume with `onResume: 'reboot'` recovers the disk state. + * * @param sandboxId sandbox ID. - * @param opts pause options, including `mode` and connection options. + * @param opts pause options, including `mode`, `force` and connection options. * - * @returns `true` if the sandbox got paused, `false` if the sandbox was already paused. + * @returns the {@link SandboxPauseResult}: `paused` is `true` if this call + * paused the sandbox and `false` if it was already paused; `forced` whether + * the snapshot was taken without the guest agent health check. + * + * @throws {@link SandboxPauseRefusedError} if the node refused the pause; the message carries the reason. */ static async pause( sandboxId: string, opts?: SandboxPauseOpts - ): Promise { + ): Promise { const apiOpts = this.resolveOpts(opts) const config = new ConnectionConfig(apiOpts) const client = new ApiClient(config) @@ -1609,6 +1655,7 @@ export class SandboxApi extends ClientFactory { }, body: { memory: snapshotMemory('mode', apiOpts?.mode, apiOpts?.keepMemory), + force: apiOpts?.force, }, signal: config.getSignal(apiOpts?.requestTimeoutMs, apiOpts?.signal), }) @@ -1619,7 +1666,7 @@ export class SandboxApi extends ClientFactory { if (res.error?.code === 409) { // Sandbox is already paused - return false + return { paused: false, forced: false } } const err = handleApiError(res) @@ -1627,7 +1674,8 @@ export class SandboxApi extends ClientFactory { throw err } - return true + // An older API answers 204 without a body. + return { paused: true, forced: res.data?.forced ?? false } } /** @@ -1636,7 +1684,7 @@ export class SandboxApi extends ClientFactory { static async betaPause( sandboxId: string, opts?: SandboxPauseOpts - ): Promise { + ): Promise { return this.pause(sandboxId, opts) } @@ -1912,7 +1960,13 @@ export class SandboxApi extends ClientFactory { `${result.error.code}: ${result.error.message}` ) } - return apiErrorFromCode(result.error.code, result.error.message) + return apiErrorFromCode( + result.error.code, + result.error.message, + undefined, + undefined, + result.error.error_code + ) } return toSandboxCreateResponse(result.sandbox) diff --git a/packages/js-sdk/tests/api/handleApiError.test.ts b/packages/js-sdk/tests/api/handleApiError.test.ts index 31ad79ba41..ef4996e6bd 100644 --- a/packages/js-sdk/tests/api/handleApiError.test.ts +++ b/packages/js-sdk/tests/api/handleApiError.test.ts @@ -3,6 +3,7 @@ import { handleApiError } from '../../src/api' import { AuthenticationError, RateLimitError, + SandboxPauseRefusedError, ServiceBusyError, SandboxError, } from '../../src/errors' @@ -67,6 +68,35 @@ describe('handleApiError', () => { assert.include(err.message, 'node is busy persisting sandbox') }) + test('a 503 carrying error_code sandbox_pause_refused is a SandboxPauseRefusedError with the API message, whatever the endpoint', () => { + const res = createMockResponse(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message: + "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry", + }) + const err = handleApiError(res as any) as SandboxPauseRefusedError + assert.instanceOf(err, SandboxPauseRefusedError) + assert.instanceOf(err, ServiceBusyError) + assert.equal(err.statusCode, 503) + assert.equal(err.errorCode, 'sandbox_pause_refused') + assert.equal( + err.message, + "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry" + ) + }) + + test('a 503 carrying another error_code stays a ServiceBusyError', () => { + const res = createMockResponse(503, { + code: 503, + error_code: 'sandbox_capacity_unavailable', + message: 'no capacity', + }) + const err = handleApiError(res as any) + assert.instanceOf(err, ServiceBusyError) + assert.notInstanceOf(err, SandboxPauseRefusedError) + }) + test('a generic failure keeps SandboxError and carries its status', () => { const res = createMockResponse(500, { message: 'boom' }) const err = handleApiError(res as any) as SandboxError diff --git a/packages/js-sdk/tests/retry.test.ts b/packages/js-sdk/tests/retry.test.ts index 7f43e9f9e7..ac808abc20 100644 --- a/packages/js-sdk/tests/retry.test.ts +++ b/packages/js-sdk/tests/retry.test.ts @@ -237,6 +237,346 @@ test.each([ expect(fetchImpl).toHaveBeenCalledTimes(calls) }) +function refusal(errorCode: string): Response { + return new Response( + JSON.stringify({ code: 503, error_code: errorCode, message: 'refused' }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) +} + +// The peek at a 503 body waits through the injected sleep too, so a sleep that +// resolves at once ends every peek before an in-memory body can be read; the +// peek tests use a real, short timer instead. +const realSleep = (delayMs: number) => + new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, 100))) + +test('returns a 503 carrying a non-retryable error_code after one request, body intact', async () => { + const fetchImpl = vi.fn(async () => + refusal('sandbox_pause_refused') + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('a 503 whose body was already consumed is not peeked at and is retried', async () => { + const consumed = refusal('sandbox_pause_refused') + await consumed.text() + const responses = [consumed, new Response('ok')] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + +test('a 503 whose body is locked is not peeked at and is retried', async () => { + const locked = refusal('sandbox_pause_refused') + const reader = locked.body!.getReader() + const responses = [locked, new Response('ok')] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + reader.releaseLock() +}) + +test('a 503 is not peeked at when the request was already aborted', async () => { + const controller = new AbortController() + const responses = [refusal('sandbox_pause_refused'), new Response('ok')] + const fetchImpl = vi.fn(async () => { + controller.abort() + return responses.shift()! + }) as typeof fetch + // A sleep that ignores the signal, so the loop is observed past the peek. + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + }) + + await fetchWithRetry('https://api.e2b.test/sandboxes/sbx-1/pause', { + method: 'POST', + signal: controller.signal, + }) + + // Not short-circuited although the body carried the code, and not peeked + // at: the only sleep is the retry delay (a peek would have added one). + expect(fetchImpl).toHaveBeenCalledTimes(2) + expect(sleep).toHaveBeenCalledTimes(1) +}) + +test('a 503 whose body stalls is retried after its inspection budget (max(delay, 1 s)), not held for the whole request budget', async () => { + const fetchImpl = vi.fn( + async () => + new Response(new ReadableStream({ start() {} }), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }) + ) as typeof fetch + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + random: () => 0, + }) + + const startedAt = performance.now() + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + // Every attempt peeks for the inspection budget (1 s, above the 50, 100, + // 200 ms backoffs) through the injected sleep, then sleeps its backoff + // (the stub clock never advances, so nothing is subtracted), and retries. + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(4) + expect(sleep.mock.calls.map(([delayMs]) => delayMs)).toEqual([ + 1000, 50, 1000, 100, 1000, 200, + ]) + expect(performance.now() - startedAt).toBeLessThan(1_000) +}) + +test('a refusal that arrives within the delay still short-circuits', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + '{"code":503,"error_code":"sandbox_pause_refused","message":"refused"}' + ) + ) + controller.close() + }, 20) + }, + }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + random: () => 1, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + // Peeked at and returned: the caller's body is untouched and still readable. + expect(response.bodyUsed).toBe(false) + expect(response.body?.locked).toBe(false) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('a refusal body arriving after the first backoff but within the inspection budget still short-circuits', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue( + new TextEncoder().encode( + '{"code":503,"error_code":"sandbox_pause_refused","message":"refused"}' + ) + ) + controller.close() + }, 120) + }, + }), + { status: 503, headers: { 'Content-Type': 'application/json' } } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: (delayMs) => + new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, 500))), + // First backoff 50 ms: shorter than the body's 120 ms. + random: () => 0, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect((await response.json()).error_code).toBe('sandbox_pause_refused') +}) + +test('the content type is matched by media type, whatever its case and parameters', async () => { + const fetchImpl = vi.fn( + async () => + new Response( + JSON.stringify({ code: 503, error_code: 'sandbox_pause_refused' }), + { + status: 503, + headers: { 'Content-Type': 'Application/JSON; charset=UTF-8' }, + } + ) + ) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) +}) + +test('a 503 whose body keeps dripping past the inspection budget is retried and the clone read is cancelled', async () => { + let served = 0 + let cancelled = 0 + const drip = () => { + let stopped = false + let pending: ReturnType | undefined + return new ReadableStream({ + pull(controller) { + return new Promise((resolve) => { + pending = setTimeout(() => { + // The source is only told about one cancel; after it the + // controller is closed and must not be enqueued into. + if (!stopped) { + served++ + controller.enqueue(new TextEncoder().encode('{"error_code":"x')) + } + resolve() + }, 5) + }) + }, + cancel() { + stopped = true + cancelled++ + clearTimeout(pending) + }, + }) + } + const fetchImpl = vi.fn( + async () => + new Response(drip(), { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }) + ) as typeof fetch + const sleep = vi.fn(async () => {}) + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep, + random: () => 0, + }) + + const startedAt = performance.now() + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + const servedAtReturn = served + + // The last response is returned without a peek (no retry left), so the + // untouched-body assertions live in the refusal-within-the-delay test. + expect(response.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(4) + expect(sleep).toHaveBeenCalledTimes(6) + expect(performance.now() - startedAt).toBeLessThan(1_000) + + // The three retried responses had both tee branches cancelled (the peek's + // reader and the discarded body), so their sources were cancelled; the + // returned one keeps the caller's branch alive. With the peek's reader + // cancelled nothing keeps pulling it: at most the tee's one-chunk + // high-water mark can still be served for the caller's branch. + await new Promise((resolve) => setTimeout(resolve, 150)) + expect(cancelled).toBe(3) + expect(served - servedAtReturn).toBeLessThanOrEqual(2) +}) + +test('a 503 whose body exceeds the size cap is a plain 503 and is retried', async () => { + const large = JSON.stringify({ + code: 503, + error_code: 'sandbox_pause_refused', + message: 'x'.repeat(9 * 1024), + }) + const responses = [ + new Response(large, { + status: 503, + headers: { 'Content-Type': 'application/json' }, + }), + new Response('ok'), + ] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry( + 'https://api.e2b.test/sandboxes/sbx-1/pause', + { method: 'POST' } + ) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + +test('retries a 503 carrying another error_code', async () => { + const responses = [ + refusal('sandbox_capacity_unavailable'), + new Response('ok'), + ] + const fetchImpl = vi.fn(async () => responses.shift()!) as typeof fetch + const fetchWithRetry = withRetry(fetchImpl, 3, 60_000, { + monotonic: () => 0, + sleep: realSleep, + }) + + const response = await fetchWithRetry('https://api.e2b.test/sandboxes', { + method: 'POST', + }) + + expect(response.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) +}) + test('retries a 503 for a non-replayable POST', async () => { const statuses = [503, 200] const fetchImpl = vi.fn( diff --git a/packages/js-sdk/tests/sandbox/apiDefaults.test.ts b/packages/js-sdk/tests/sandbox/apiDefaults.test.ts index 3f98484460..9c7b457139 100644 --- a/packages/js-sdk/tests/sandbox/apiDefaults.test.ts +++ b/packages/js-sdk/tests/sandbox/apiDefaults.test.ts @@ -42,7 +42,7 @@ const server = setupMockApi( }), http.post(apiUrl('/sandboxes/:sandboxID/pause'), async ({ request }) => { lastPauseBody = (await request.json()) as Record - return new HttpResponse(null, { status: 204 }) + return HttpResponse.json({ forced: lastPauseBody.force === true }) }), http.post(apiUrl('/sandboxes/:sandboxID/snapshots'), async ({ request }) => { lastSnapshotBody = (await request.json()) as Record @@ -140,6 +140,33 @@ test('Sandbox.pause omits memory when mode is unset', async () => { expect(lastPauseBody).not.toHaveProperty('memory') }) +test('Sandbox.pause omits force when unset', async () => { + await Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + + expect(lastPauseBody).toBeDefined() + expect(lastPauseBody).not.toHaveProperty('force') +}) + +test.each([true, false])( + 'Sandbox.pause sends an explicit force %s', + async (force) => { + await Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force }) + + expect(lastPauseBody?.force).toBe(force) + } +) + +test('Sandbox.pause sends force alongside mode', async () => { + await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + mode: 'full', + force: true, + }) + + expect(lastPauseBody?.memory).toBe(true) + expect(lastPauseBody?.force).toBe(true) +}) + test.each([ ['filesystem', false], ['full', true], diff --git a/packages/js-sdk/tests/sandbox/pause.test.ts b/packages/js-sdk/tests/sandbox/pause.test.ts new file mode 100644 index 0000000000..bec4ee663f --- /dev/null +++ b/packages/js-sdk/tests/sandbox/pause.test.ts @@ -0,0 +1,180 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'vitest' +import { http, HttpResponse } from 'msw' + +import { + Sandbox, + SandboxError, + SandboxNotFoundError, + SandboxPauseRefusedError, + ServiceBusyError, +} from '../../src' +import { TEST_API_KEY, apiUrl } from '../setup' +import { setupMockApi } from '../mockApi' + +const server = setupMockApi() + +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) + +afterAll(() => server.close()) + +afterEach(() => { + requests = 0 + server.resetHandlers() +}) + +let requests = 0 + +function pauseAnswers(status: number, body?: unknown) { + server.use( + http.post(apiUrl('/sandboxes/:sandboxID/pause'), () => { + requests++ + return body === undefined + ? new HttpResponse(null, { status }) + : HttpResponse.json(body, { status }) + }) + ) +} + +test('a 200 with forced false resolves to paused, not forced', async () => { + pauseAnswers(200, { forced: false }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('a 200 with forced true resolves to paused and forced', async () => { + pauseAnswers(200, { forced: true }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force: true }) + ).resolves.toEqual({ paused: true, forced: true }) +}) + +test('forced reports what the API did, not the option: a forced request that joined a normal pause is not forced', async () => { + pauseAnswers(200, { forced: false }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY, force: true }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('a 204 from an older API resolves to paused, not forced', async () => { + pauseAnswers(204) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: true, forced: false }) +}) + +test('an already paused sandbox (409) resolves to not paused', async () => { + pauseAnswers(409, { code: 409, message: 'Sandbox is already paused' }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).resolves.toEqual({ paused: false, forced: false }) +}) + +test('the instance method returns the same result', async () => { + server.use( + http.post(apiUrl('/v2/sandboxes/:sandboxID/connect'), () => + HttpResponse.json({ + sandboxID: 'test-sandbox-id', + templateID: 'base', + envdVersion: '0.2.4', + }) + ) + ) + pauseAnswers(200, { forced: true }) + const sandbox = await Sandbox.connect('test-sandbox-id', { + apiKey: TEST_API_KEY, + }) + + await expect(sandbox.pause({ force: true })).resolves.toEqual({ + paused: true, + forced: true, + }) +}) + +test('a refused pause (503 sandbox_pause_refused) is a SandboxPauseRefusedError carrying the API message, after one request', async () => { + const message = + "Sandbox 'test-sandbox-id' cannot be paused right now because the sandbox guest agent is not responding: retry, or pause with force to take the snapshot without the health check" + pauseAnswers(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message, + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxPauseRefusedError) + // Subclass of the generic 503 error, so handlers written for it keep working. + expect(err).toBeInstanceOf(ServiceBusyError) + expect(err.name).toBe('SandboxPauseRefusedError') + expect(err.statusCode).toBe(503) + expect(err.errorCode).toBe('sandbox_pause_refused') + // The API's message verbatim: it names the reason and what to do. + expect(err.message).toBe(message) + // A coded refusal is not replayed by the transport retries. + expect(requests).toBe(1) +}) + +test('a refusal for a busy node carries the same code and the reason in the message', async () => { + const message = + "Sandbox 'test-sandbox-id' cannot be paused right now because its node is busy, please retry" + pauseAnswers(503, { + code: 503, + error_code: 'sandbox_pause_refused', + message, + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxPauseRefusedError) + expect(err.message).toBe(message) +}) + +test('a 503 with no error_code at all stays a plain ServiceBusyError', async () => { + pauseAnswers(503, { + code: 503, + message: + "Sandbox 'test-sandbox-id' cannot be paused right now because the server is shutting down, please retry", + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + }).catch((e) => e) + + expect(err).toBeInstanceOf(ServiceBusyError) + expect(err).not.toBeInstanceOf(SandboxPauseRefusedError) + expect(err.message).toContain('server is shutting down') +}) + +test('force on a filesystem-only pause (400) is a SandboxError carrying the status and message', async () => { + pauseAnswers(400, { + code: 400, + message: 'force applies to memory pauses only', + }) + + const err = await Sandbox.pause('test-sandbox-id', { + apiKey: TEST_API_KEY, + mode: 'filesystem', + force: true, + }).catch((e) => e) + + expect(err).toBeInstanceOf(SandboxError) + expect(err.statusCode).toBe(400) + expect(err.message).toContain('force applies to memory pauses only') +}) + +test('a missing sandbox (404) is a SandboxNotFoundError', async () => { + pauseAnswers(404, { code: 404, message: 'Sandbox not found' }) + + await expect( + Sandbox.pause('test-sandbox-id', { apiKey: TEST_API_KEY }) + ).rejects.toBeInstanceOf(SandboxNotFoundError) +}) From 891dec7ad29cbfa61ac3da9edb54a11b6c0addfb Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Wed, 14 Oct 2026 23:38:26 +0000 Subject: [PATCH 3/3] feat(python-sdk)!: pause takes force and returns a SandboxPauseResult; SandboxPauseRefusedException for a refused pause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What changed: `Sandbox.pause()` and `AsyncSandbox.pause()` (instance and class forms, and the deprecated `beta_pause` forms, matching the JavaScript `betaPause`) take `force: Optional[bool] = None`, sent in the pause request body when set and omitted otherwise, like `mode`, and return the `SandboxPauseResult` dataclass (`paused`, `forced`), exported from `e2b` under its own section, instead of a bool: 200 gives `paused=True, forced=body.forced`, 409 (already paused) `paused=False, forced=False`, and a 204 from an older API `paused=True, forced=False`. A 503 whose body carries `error_code: "sandbox_pause_refused"` is raised as the new `SandboxPauseRefusedException` (exported; `status_code` 503, the `error_code` attribute, subclass of `ServiceBusyException` so existing handlers keep working). The mapping lives in `api_exception_from_code` / `handle_api_exception`, keyed on the body's `error_code` and passed through for errors embedded in fork results too, so any endpoint answering the code gets the typed exception; a non-object JSON body (a list) carries neither field and maps as before; the pause call site keeps only the 409 → `paused=False` case. The message is the API's verbatim: belt's messages already name the reason and what to do for it, so the SDK appends nothing and the general guidance lives in the class docstring. A 503 without the code stays a `ServiceBusyException`, 404 raises `SandboxNotFoundException`, and 400 (force on a filesystem-only pause) is the generic `SandboxException` carrying the status. The code string is one constant, `SANDBOX_PAUSE_REFUSED_ERROR_CODE`, exported from `e2b` and read by the exception class, the mapping and the retry predicate. `RetryableTransport` / `AsyncRetryableTransport`, which retry 503s the same way the JavaScript transport does, peek at the body only for a 503 whose media type is JSON (lower-cased, parameters dropped) and that is not declared larger than 8 KiB, streaming it through iter_bytes / aiter_bytes under the existing try with that cap and a wall-clock budget of min(max(delay, 1 s), remaining request budget) measured with the transport's monotonic clock across chunks (each chunk is also bounded by the request's read timeout as pyqwest applies it, none when the request timeout is disabled; a chunk whose wait began within the budget is kept, only waiting for more past it gives up), so a chunked or HTTP/2 refusal without Content-Length is read and a trickling body cannot hold the loop: a read that fails, times out, exceeds the cap or outlives the budget decides nothing and the 503 is closed and retried, where an earlier draft let the exception escape after one request; the time the read took is subtracted from the following sleep and the deadline re-checked, as the JavaScript side does; with no retry budget left, the 503 is the outcome as on the JavaScript side: a body read in full is returned as is, one stopped at the byte cap or cut off by the peek budget is handed on whole (the peeked chunks first, then the rest of the stream, through a resumed stream whose close releases the body iterator and the original response), and a failed read re-raises its error, never a fabricated body (the API client parses every 503 body as JSON). While budget remains, the iterator a stopped peek left is closed explicitly before the retry, so nothing is left to garbage collection; the rebuilt response is constructed once, with httpx taking the stream over the content when both are given. A body nested too deep for json.loads (RecursionError, not ValueError) is not a refusal; a BaseException out of the peek (cancellation, KeyboardInterrupt) closes the inner response before propagating, so the connection is not leaked; the resumed stream closes the inner response in a finally and tolerates a body generator a reader is still running (RuntimeError for an async generator, ValueError for a sync one; left to that reader). An ``error_code`` that is not a string (a list or object from a proxy's or a buggy server's body) is not a refusal rather than a TypeError out of the transport. A peek cut off by the budget on what turns out to be the last chunk holds one complete JSON document: it is treated as the body read in full (a refusal is returned at once; any other complete API error follows the normal path, retried while budget remains and returned whole when it is exhausted) and the tail is left unread; only a body that does not yet parse is handed on. Without this a busy 503 whose body completed in the cut-off chunk surfaced as ReadTimeout instead of ServiceBusyException. The handed-on stream's live tail is bounded by the request deadline: a chunk whose wait would begin past it is not waited for, the inner response is closed and ReadTimeout is raised, so the API client's read ends at the deadline instead of running on chunk by chunk past the request timeout. Because the peek's time counts against the retry delay, a budget exhausted after a peek means the clock is at the deadline, so in practice an over-cap or cut-off body handed on at that point yields what the peek buffered and then the timeout; a body read in full, or buffered chunks that already form the refusal, are returned as content and never reach the tail. The two readers' cross-close tests therefore build the resumed streams directly with the deadline ahead. A refusal is returned after one request as a response rebuilt from the bytes read: the transport's response carries no request yet (httpx attaches it after the transport returns; pyqwest's arrive without one), so the handled request is attached explicitly, Content-Encoding, Transfer-Encoding and Content-Length are dropped (the content is decoded and complete; httpx sets the length), and only plain extensions such as http_version are copied, never pyqwest's get_trailers bound to the closed native response. The earlier draft read response.request and raised RuntimeError on the production transport for every refused pause; the pause tests missed it because their fakes attached request= and bypassed the retry layer. `@e2b/code-interpreter`, `@e2b/desktop`, `@e2b/code-interpreter-python` and `@e2b/desktop-python` take the same major in the changeset: the JavaScript two re-export `e2b` and subclass its Sandbox, the Python two subclass the Python Sandbox, and `updateInternalDependencies: patch` would otherwise ship the breaking `pause()` in a patch of theirs. The two Python dependents' `e2b` bound is raised from `<3.0.0` to `<4.0.0` so they keep resolving against e2b 3.x (their locks are unchanged: they resolve the workspace e2b). `pnpm changeset status --verbose` lists exactly those six majors and the CLI minor. Docstrings on every overload and on the exception describe the refusal as the node refusing the pause with the sandbox kept running and the reason in the message, and present force as the answer for the unresponsive-guest reason only, with the reboot caveat. The two live tests that asserted the old bool read `.paused`. The changeset is major for both SDKs (minor for the CLI) with the migration note. Why: the API now refuses a memory pause when the sandbox guest agent is unresponsive and keeps the sandbox running; callers need to tell that apart from a busy node (retry) and need the override to snapshot anyway, accepting that the memory may not resume (a resume with `on_resume="reboot"` recovers the disk state). The result object is a breaking change, decided by the operator: `forced` is informative, not an echo of the option (the API reports `forced=False` when a forced request joined a normal pause already in flight). `force` is `Optional[bool]` rather than `bool = False` so an unset flag is omitted from the payload (#1749: no SDK-side defaults in API request bodies). The generated 200 model is imported as `SandboxPauseResultAPI` to keep the public dataclass name. Verification: unit tests with the generated client mocked cover force omitted / true / false / with mode in the body; 200 with forced False and True; forced reported False for a forced request that joined a normal pause; 204 compatibility; 409; the coded refusal raising SandboxPauseRefusedException (a ServiceBusyException) with the API message verbatim; the busy-node refusal; a 503 with no error_code staying ServiceBusyException; 400 carrying the status; 404, for sync and async. Tests over an httpx.MockTransport-backed ApiClient / AsyncApiClient now run through RetryableTransport / AsyncRetryableTransport with responses built as httpx and pyqwest hand them over (no request=): the refusal-through-real-parsing tests fail on the earlier draft with RuntimeError and pass here; they also cover 200 with forced True and False, the 204 older-API path (parsed is None) and, through Sandbox.pause, a refusal body read that times out with no budget left surfacing as the timeout. The transport fakes lost their request= too. The transport tests assert a coded 503 is returned after one request with its body intact, a chunked refusal without Content-Length is returned after one request with its body readable (sync and async), an oversized chunked body and a non-JSON body are retried, a trickling body is retried once the peek budget is spent (sync and async), a complete but slow body read with no budget left is returned with its body (sync and async), an over-cap body with no budget left surfaces the 503 whole with its third chunk read only by the resumed stream and no length or transfer-encoding header (sync and async, and through Sandbox.pause as SandboxPauseRefusedException), a read cut off by the budget with no budget left is handed on whole too (sync and async, and through Sandbox.pause as the typed refusal), a 3000-deep JSON body is retried (sync and async), cancelling the peek while it awaits a chunk closes the inner response exactly once, and closing the handed-on response while a reader awaits its live tail closes the inner exactly once (an asyncio task, and a thread inside iter_bytes), a list-valued error_code is retried (sync and async), and a cut-off peek whose buffered chunks form the refusal returns it without retrying and without pulling the tail (sync and async, and through Sandbox.pause and AsyncSandbox.pause), a busy 503 whose complete body lands in the cut-off chunk is retried while budget remains and surfaces as ServiceBusyException through Sandbox.pause and AsyncSandbox.pause when none is left, the over-cap deadline cut is covered through AsyncSandbox.pause too, a handed-on tail is cut at the deadline with ReadTimeout and the inner closed once (transport sync and async, and through Sandbox.pause for both the cut-off and the over-cap case), a failed read with no budget left re-raises (sync and async), a mixed-case Content-Type is matched by media type, the read time counts towards the delay, the rebuilt refusal drops the transfer headers and native extensions and carries the request, a 503 with another code is still retried, sync and async. `ruff check`, `ruff format --check`, `ty check` and the non-live pytest suite pass (670 tests); `make lint` exits 0 and a second `ruff format` leaves every file unchanged. --- .changeset/pause-force.md | 17 + .../code-interpreter-python/pyproject.toml | 2 +- packages/desktop-python/pyproject.toml | 2 +- packages/python-sdk/e2b/__init__.py | 7 + packages/python-sdk/e2b/api/__init__.py | 22 +- packages/python-sdk/e2b/exceptions.py | 31 + packages/python-sdk/e2b/retry.py | 405 +++++- .../python-sdk/e2b/sandbox/sandbox_api.py | 18 + packages/python-sdk/e2b/sandbox_async/main.py | 39 +- .../e2b/sandbox_async/sandbox_api.py | 28 +- packages/python-sdk/e2b/sandbox_sync/main.py | 39 +- .../e2b/sandbox_sync/sandbox_api.py | 28 +- .../test_snapshot_filesystem_only.py | 2 +- .../tests/shared/sandbox/test_api_defaults.py | 41 +- .../tests/shared/sandbox/test_pause.py | 756 +++++++++++ .../test_snapshot_filesystem_only.py | 2 +- .../python-sdk/tests/test_api_exception.py | 38 + .../tests/test_rate_limit_retry_transport.py | 1208 ++++++++++++++++- 18 files changed, 2640 insertions(+), 45 deletions(-) create mode 100644 .changeset/pause-force.md create mode 100644 packages/python-sdk/tests/shared/sandbox/test_pause.py diff --git a/.changeset/pause-force.md b/.changeset/pause-force.md new file mode 100644 index 0000000000..2d3ad7d493 --- /dev/null +++ b/.changeset/pause-force.md @@ -0,0 +1,17 @@ +--- +'e2b': major +'@e2b/python-sdk': major +'@e2b/code-interpreter': major +'@e2b/desktop': major +'@e2b/code-interpreter-python': major +'@e2b/desktop-python': major +'@e2b/cli': minor +--- + +`pause()` now returns a result object instead of a boolean, and takes `force`. + +**Breaking:** `Sandbox.pause()` / `SandboxApi.pause()` (JavaScript) and `Sandbox.pause()` / `AsyncSandbox.pause()` (Python) return `SandboxPauseResult` with `paused` (`true` if this call paused the sandbox, `false` if it was already paused) and `forced` (whether the snapshot was taken without the guest agent health check, as reported by the API). Migrate by reading `paused` where the boolean was read: `if (await sandbox.pause())` becomes `if ((await sandbox.pause()).paused)` and `if sandbox.pause():` becomes `if sandbox.pause().paused:`. A plain truthiness check on the result is now always true, so update every call site that branched on the old boolean. `@e2b/code-interpreter` and `@e2b/desktop` re-export `e2b` and subclass its `Sandbox`, and `e2b-code-interpreter` / `e2b-desktop` subclass the Python `Sandbox`, so their `pause()` changes the same way and they take the same major; the Python packages now allow `e2b<4.0.0`. + +The node may refuse a pause (HTTP 503 with `error_code: "sandbox_pause_refused"`): the sandbox keeps running and the refusal surfaces as `SandboxPauseRefusedError` (JavaScript) / `SandboxPauseRefusedException` (Python), a subclass of `ServiceBusyError` / `ServiceBusyException` whose message carries the reason. When the reason is an unresponsive sandbox guest agent, the new `force` option (`{ force: true }` / `force=True`) takes the snapshot without the health check; its memory may then not resume, and a resume with `onResume: 'reboot'` / `on_resume="reboot"` recovers the disk state. Unlike other 503s, a refusal carrying this code is not replayed by the SDK's transport retries, in either SDK: it surfaces after a single request. A `204` from an older API is still a successful, unforced pause. + +`e2b sandbox pause` takes `-f, --force` and says when the pause was forced. `SANDBOX_PAUSE_REFUSED_ERROR_CODE` is exported from both packages. diff --git a/packages/code-interpreter-python/pyproject.toml b/packages/code-interpreter-python/pyproject.toml index c86020a5df..cf4773d7d7 100644 --- a/packages/code-interpreter-python/pyproject.toml +++ b/packages/code-interpreter-python/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.11" dependencies = [ "httpx>=0.20.0,<1.0.0", "attrs>=21.3.0", - "e2b>=2.55.1,<3.0.0", + "e2b>=2.55.1,<4.0.0", ] [project.urls] diff --git a/packages/desktop-python/pyproject.toml b/packages/desktop-python/pyproject.toml index 6c91abd6d4..595408ac72 100644 --- a/packages/desktop-python/pyproject.toml +++ b/packages/desktop-python/pyproject.toml @@ -7,7 +7,7 @@ license = "MIT" readme = "README.md" requires-python = ">=3.11" dependencies = [ - "e2b>=2.44.0,<3.0.0", + "e2b>=2.44.0,<4.0.0", "requests>=2.32.3,<3", "urllib3>=2.8.0,<3", "pillow>=12.0.0,<13", diff --git a/packages/python-sdk/e2b/__init__.py b/packages/python-sdk/e2b/__init__.py index b4f1d0984a..7afac418cc 100644 --- a/packages/python-sdk/e2b/__init__.py +++ b/packages/python-sdk/e2b/__init__.py @@ -38,6 +38,7 @@ ) from .volume.connection_config import VolumeApiParams, VolumeConnectionConfig from .exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, AuthenticationException, FileNotFoundException, GitAuthException, @@ -50,6 +51,7 @@ RateLimitException, ServiceBusyException, SandboxException, + SandboxPauseRefusedException, SandboxNotFoundException, SandboxNotRunningException, SandboxUnreachableException, @@ -93,6 +95,7 @@ SandboxMetrics, SandboxLifecycle, SandboxOnTimeout, + SandboxPauseResult, SandboxNetworkInfo, SandboxNetworkOpts, SandboxNetworkRule, @@ -184,6 +187,8 @@ # Exceptions "SandboxException", "ServiceBusyException", + "SandboxPauseRefusedException", + "SANDBOX_PAUSE_REFUSED_ERROR_CODE", "TimeoutException", "NotFoundException", "FileNotFoundException", @@ -258,6 +263,8 @@ "AsyncSecretPaginator", "SecretException", "SecretNotFoundException", + # Pause + "SandboxPauseResult", # Snapshot "SnapshotInfo", "SnapshotMode", diff --git a/packages/python-sdk/e2b/api/__init__.py b/packages/python-sdk/e2b/api/__init__.py index 68f7d9f243..a58d89ae7e 100644 --- a/packages/python-sdk/e2b/api/__init__.py +++ b/packages/python-sdk/e2b/api/__init__.py @@ -21,9 +21,11 @@ from e2b.api.metadata import default_headers from e2b.connection_config import ConnectionConfig, ProxyTypes from e2b.exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, AuthenticationException, InvalidArgumentException, RateLimitException, + SandboxPauseRefusedException, ServiceBusyException, SandboxException, ) @@ -157,10 +159,13 @@ def api_exception_from_code( message: Optional[str] = None, default_exception_class: type[Exception] = SandboxException, stack_trace: Optional[TracebackType] = None, + error_code: Optional[str] = None, ) -> Exception: """Map an API error code and message to the matching exception class — the same mapping :func:`handle_api_exception` applies to HTTP responses, usable - for error objects embedded in response bodies (e.g. per-fork results).""" + for error objects embedded in response bodies (e.g. per-fork results). + ``error_code`` is the body's semantic error code, which refines the status + for the codes the SDK types (a refused pause).""" if status_code == 401: text = f"{status_code}: Unauthorized, please check your credentials." if message: @@ -174,6 +179,11 @@ def api_exception_from_code( return RateLimitException(text) if status_code == 503: + if error_code == SANDBOX_PAUSE_REFUSED_ERROR_CODE: + # The API's message already says what to do for its reason. + return SandboxPauseRefusedException( + message or "The node refused the pause, please retry" + ) text = f"{status_code}: Service temporarily unavailable, please retry." if message: text += f" - {message}" @@ -197,8 +207,12 @@ def handle_api_exception( body = json.loads(e.content) if e.content else {} except json.JSONDecodeError: body = {} + # A non-object body (e.g. a JSON list) carries neither field. + if not isinstance(body, dict): + body = {} message = body["message"] if "message" in body else None + error_code = body.get("error_code") if message is None and e.status_code not in (401, 429, 503): err = default_exception_class(f"{e.status_code}: {e.content}").with_traceback( stack_trace @@ -209,7 +223,11 @@ def handle_api_exception( return err return api_exception_from_code( - e.status_code, message, default_exception_class, stack_trace + e.status_code, + message, + default_exception_class, + stack_trace, + error_code if isinstance(error_code, str) else None, ) diff --git a/packages/python-sdk/e2b/exceptions.py b/packages/python-sdk/e2b/exceptions.py index d479e21313..914a2a7790 100644 --- a/packages/python-sdk/e2b/exceptions.py +++ b/packages/python-sdk/e2b/exceptions.py @@ -223,6 +223,37 @@ class ServiceBusyException(Exception): status_code = 503 +# ``error_code`` of the 503 the API answers a pause with when the node refused +# it. Any endpoint answering this code is mapped to +# ``SandboxPauseRefusedException``. +SANDBOX_PAUSE_REFUSED_ERROR_CODE = "sandbox_pause_refused" + + +class SandboxPauseRefusedException(ServiceBusyException): + """ + Raised when the node refused a pause (HTTP 503 with + ``error_code: "sandbox_pause_refused"``). The sandbox keeps running with its + state intact; the message is the API's, and names the reason and what to + do about it. In general: retry, and when the reason is an unresponsive + sandbox guest agent, pause with ``force=True`` to take the snapshot without + the health check -- its memory may then not resume, and a resume with + ``on_resume="reboot"`` recovers the disk state. + + Unlike other 503s, a refusal carrying this code is not replayed by the + SDK's transport retries: it is a decision, not transient capacity, so it + surfaces after a single request and the caller decides what to do. + + Subclass of `ServiceBusyException`, which a refused pause surfaced as + before, so existing handlers keep working. Unlike the generic 503 mapping, + the message carries no ``503: `` prefix: it is the API's message verbatim. + + The ``error_code`` attribute is the semantic error code of the API + response, always ``sandbox_pause_refused``. + """ + + error_code = SANDBOX_PAUSE_REFUSED_ERROR_CODE + + class BuildException(Exception): """ Raised when the build fails. diff --git a/packages/python-sdk/e2b/retry.py b/packages/python-sdk/e2b/retry.py index 50510fccfa..5e8b0f1560 100644 --- a/packages/python-sdk/e2b/retry.py +++ b/packages/python-sdk/e2b/retry.py @@ -1,12 +1,25 @@ import asyncio +import json import random import re import time -from typing import Awaitable, Callable, List, Optional, Tuple +from dataclasses import dataclass, field +from collections.abc import AsyncIterator, Iterator +from typing import ( + Awaitable, + Callable, + List, + Optional, + Tuple, + Union, +) import httpx -from e2b.exceptions import InvalidArgumentException +from e2b.exceptions import ( + SANDBOX_PAUSE_REFUSED_ERROR_CODE, + InvalidArgumentException, +) MAX_RETRY_AFTER_SECONDS = 2_147_483_647 MAX_RETRY_WAIT_WITHOUT_TIMEOUT_SECONDS = 60.0 @@ -14,6 +27,16 @@ BACKOFF_MAX_SECONDS = 10.0 BACKOFF_JITTER_MIN = 0.5 RETRYABLE_STATUSES = frozenset({429, 502, 503}) +# A 503 whose body carries one of these ``error_code`` values is a decision by +# the server, not transient capacity, and is answered after a single request. +NON_RETRYABLE_ERROR_CODES = frozenset({SANDBOX_PAUSE_REFUSED_ERROR_CODE}) +# An error body larger than this cannot carry a code worth parsing, so the peek +# stops there: such a 503 is retried while budget remains and handed on whole +# when it is exhausted. +MAX_ERROR_BODY_BYTES = 8 * 1024 +# Wall-clock cap on peeking at a 503 body for its ``error_code``, measured +# across chunks; the JavaScript transport's PEEK_BUDGET_MS counterpart. +PEEK_BUDGET_SECONDS = 1.0 # GET/PUT/PATCH/DELETE are idempotent by HTTP semantics. A POST is retried # after a 502 or a network error that may have occurred once the request was @@ -104,6 +127,263 @@ def _retry_delay( return _backoff_delay(attempt, random_) +def _media_type(content_type: Optional[str]) -> str: + """The media type of a ``Content-Type`` value: lower-cased, parameters + dropped.""" + return (content_type or "").split(";", 1)[0].strip().lower() + + +def _should_read_error_body(response: httpx.Response) -> bool: + """Whether a 503 body is worth reading for its ``error_code``: the status + matches, the body is JSON, and it is not declared larger than + ``MAX_ERROR_BODY_BYTES``. The read itself streams up to that cap (so a + chunked or HTTP/2 body without ``Content-Length`` is read too). Each chunk + is bounded by the request's read timeout as the transport applies it + (pyqwest's operation timeout; none when the request timeout is disabled), + and the whole peek by a wall-clock budget measured across chunks (see + ``_read_error_body``). A read that fails, exceeds the cap or outlives the + budget decides nothing: the 503 is retried while budget remains; with none + left, a body past the cap or cut off by the budget is handed on whole and + a failed read is re-raised.""" + if response.status_code != 503: + return False + if _media_type(response.headers.get("Content-Type")) != "application/json": + return False + length = response.headers.get("Content-Length") + return not ( + length is not None and length.isdecimal() and int(length) > MAX_ERROR_BODY_BYTES + ) + + +@dataclass +class _Peek: + """What peeking at a 503 body yielded: ``content`` is the whole body when + it was read in full; otherwise ``buffered`` holds the chunks read before + the peek stopped (at the byte cap or past the wall-clock budget) and + ``rest`` the live iterator over the remainder, so the body can be handed + on whole or released.""" + + content: Optional[bytes] = None + buffered: List[bytes] = field(default_factory=list) + rest: Union[Iterator[bytes], AsyncIterator[bytes], None] = None + + +def _close_rest(peek: _Peek) -> None: + """Release a peek's remaining body iterator (httpx hands out a generator) + now rather than at garbage collection. A generator a reader is still + running cannot be closed from outside; it is left to that reader.""" + close = getattr(peek.rest, "close", None) + if close is not None: + try: + close() + except (RuntimeError, ValueError): + # "generator already executing": a reader owns it. + pass + + +async def _aclose_rest(peek: _Peek) -> None: + """Async counterpart of :func:`_close_rest`.""" + aclose = getattr(peek.rest, "aclose", None) + if aclose is not None: + try: + await aclose() + except (RuntimeError, ValueError): + # "asynchronous generator is already running": a reader owns it. + pass + + +def _read_error_body( + response: httpx.Response, until: float, monotonic: Callable[[], float] +) -> _Peek: + """Up to ``MAX_ERROR_BODY_BYTES`` of the (decoded) body. Stops, keeping + the stream resumable, at the byte cap or once a wait for a further chunk + began after ``until`` (``monotonic()``). A chunk whose wait began within + the budget is kept whenever it arrives, so a complete body is never + dropped for being slow; only waiting for more past the budget gives up.""" + chunks: List[bytes] = [] + size = 0 + stream = response.iter_bytes() + while True: + waited_from = monotonic() + chunk = next(stream, None) + if chunk is None: + return _Peek(content=b"".join(chunks)) + chunks.append(chunk) + size += len(chunk) + if size > MAX_ERROR_BODY_BYTES or waited_from > until: + return _Peek(buffered=chunks, rest=stream) + + +async def _aread_error_body( + response: httpx.Response, until: float, monotonic: Callable[[], float] +) -> _Peek: + """Async counterpart of :func:`_read_error_body`.""" + chunks: List[bytes] = [] + size = 0 + stream = response.aiter_bytes() + while True: + waited_from = monotonic() + try: + chunk = await stream.__anext__() + except StopAsyncIteration: + return _Peek(content=b"".join(chunks)) + chunks.append(chunk) + size += len(chunk) + if size > MAX_ERROR_BODY_BYTES or waited_from > until: + return _Peek(buffered=chunks, rest=stream) + + +_TAIL_TIMEOUT_MESSAGE = ( + "Request timed out while reading the response body. " + "Increase `request_timeout` or lower `retries`." +) + + +class _ResumedStream(httpx.SyncByteStream): + """The peeked chunks followed by the rest of the original body, the tail + bounded by the request deadline: a chunk whose wait would begin past it + is not waited for; the inner response is closed and ``ReadTimeout`` is + raised, so the client's read ends at the deadline instead of running on + chunk by chunk. Since the peek's time counts against the retry delay, a + budget exhausted after a peek means the clock is at the deadline, so in + practice the caller gets what the peek buffered and then the timeout; a + body the peek read in full, or buffered chunks that already form the + refusal, are returned as content instead and never reach this stream.""" + + def __init__( + self, + response: httpx.Response, + peek: _Peek, + request: httpx.Request, + deadline: float, + monotonic: Callable[[], float], + ): + self._response = response + self._peek = peek + self._request = request + self._deadline = deadline + self._monotonic = monotonic + + def __iter__(self) -> Iterator[bytes]: + yield from self._peek.buffered + rest = self._peek.rest + if not isinstance(rest, Iterator): + return + while True: + if self._monotonic() >= self._deadline: + self.close() + raise httpx.ReadTimeout(_TAIL_TIMEOUT_MESSAGE, request=self._request) + chunk = next(rest, None) + if chunk is None: + return + yield chunk + + def close(self) -> None: + try: + _close_rest(self._peek) + finally: + self._response.close() + + +class _AsyncResumedStream(httpx.AsyncByteStream): + def __init__( + self, + response: httpx.Response, + peek: _Peek, + request: httpx.Request, + deadline: float, + monotonic: Callable[[], float], + ): + self._response = response + self._peek = peek + self._request = request + self._deadline = deadline + self._monotonic = monotonic + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._peek.buffered: + yield chunk + rest = self._peek.rest + if not isinstance(rest, AsyncIterator): + return + while True: + if self._monotonic() >= self._deadline: + await self.aclose() + raise httpx.ReadTimeout(_TAIL_TIMEOUT_MESSAGE, request=self._request) + try: + chunk = await rest.__anext__() + except StopAsyncIteration: + return + yield chunk + + async def aclose(self) -> None: + try: + await _aclose_rest(self._peek) + finally: + await self._response.aclose() + + +# Response extensions that are plain values and survive rebuilding the response; +# pyqwest's ``get_trailers`` is bound to the native response and is not copied. +_REBUILT_EXTENSIONS = ("http_version", "reason_phrase") + + +def _rebuild( + response: httpx.Response, + request: httpx.Request, + *, + content: Optional[bytes] = None, + stream: Union[httpx.SyncByteStream, httpx.AsyncByteStream, None] = None, +) -> httpx.Response: + """The response with its (partly) consumed stream replaced by ``content`` + or ``stream`` (httpx takes the stream when both are given), both decoded, + so the encoding, transfer-encoding and length headers are dropped (httpx + sets the length of a content body). ``request`` is the one being handled: + the transport's response carries none yet, httpx attaches it after the + transport returns.""" + headers = [ + (name, value) + for name, value in response.headers.raw + if name.lower() + not in (b"content-encoding", b"transfer-encoding", b"content-length") + ] + return httpx.Response( + response.status_code, + headers=headers, + content=content, + stream=stream, + request=request, + extensions={ + key: value + for key, value in response.extensions.items() + if key in _REBUILT_EXTENSIONS + }, + ) + + +def _is_complete_json(content: bytes) -> bool: + """Whether ``content`` parses as one complete JSON document.""" + try: + json.loads(content) + except (ValueError, RecursionError): + return False + return True + + +def _is_non_retryable_refusal(content: bytes) -> bool: + """Whether a 503 body carries a non-retryable ``error_code`` (see + ``NON_RETRYABLE_ERROR_CODES``).""" + try: + body = json.loads(content) + except (ValueError, RecursionError): + # Not JSON, or nested too deep to parse: not a refusal either way. + return False + if not isinstance(body, dict): + return False + code = body.get("error_code") + return isinstance(code, str) and code in NON_RETRYABLE_ERROR_CODES + + def _copy_request( request: httpx.Request, remaining_timeout: Optional[float] = None ) -> httpx.Request: @@ -142,7 +422,9 @@ class RetryableTransport(httpx.BaseTransport): """Retry requests after a 429 carrying ``Retry-After`` or a 503, and — for replayable operations (``is_replayable``) — after a 502 (using ``Retry-After`` when present, exponential backoff otherwise) or a network - error once the request was written.""" + error once the request was written. A 503 whose body carries a + non-retryable ``error_code`` (a refused pause) is returned after one + request.""" def __init__( self, @@ -196,6 +478,63 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: delay = _retry_delay(response, attempt, replayable, self._random) if delay is None or self._monotonic() + delay >= deadline: return response + if _should_read_error_body(response): + read_started = self._monotonic() + peek_until = min( + read_started + max(delay, PEEK_BUDGET_SECONDS), deadline + ) + peek = _Peek() + read_error: Optional[Exception] = None + try: + peek = _read_error_body(response, peek_until, self._monotonic) + except (httpx.HTTPError, httpx.StreamError) as exc: + # A body that cannot be read decides nothing: retry as before. + read_error = exc + except BaseException: + # Interrupted mid-peek: do not leak the connection. + response.close() + raise + if peek.content is None and peek.rest is not None: + # Cut off on what was the last chunk: the buffered bytes are one + # complete JSON document, so they are the body read in full (a + # refusal or any other complete API error) and the tail is left + # unread; only a body that does not yet parse is handed on. + buffered = b"".join(peek.buffered) + if len(buffered) <= MAX_ERROR_BODY_BYTES and _is_complete_json( + buffered + ): + _close_rest(peek) + peek = _Peek(content=buffered) + if peek.content is not None and _is_non_retryable_refusal(peek.content): + response.close() + return _rebuild(response, request, content=peek.content) + # The time spent reading counts towards the delay. + delay = max(0.0, delay - (self._monotonic() - read_started)) + if self._monotonic() + delay >= deadline: + # Out of budget: the 503 is the outcome, as on the + # JavaScript side. A body read in full is returned as is; + # one stopped at the byte cap or cut off by the peek budget + # is handed on whole, the peeked chunks first and the rest + # of the stream after; a failed read is reported as such, + # never as a fabricated body (the API client parses every + # 503 body as JSON). + if peek.content is not None: + response.close() + return _rebuild(response, request, content=peek.content) + if peek.rest is not None: + return _rebuild( + response, + request, + stream=_ResumedStream( + response, peek, request, deadline, self._monotonic + ), + ) + response.close() + if read_error is not None: + raise read_error + raise AssertionError("unreachable") + # Retrying: release what the peek left of the body. + _close_rest(peek) response.close() self._sleep(delay) @@ -262,6 +601,66 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: delay = _retry_delay(response, attempt, replayable, self._random) if delay is None or self._monotonic() + delay >= deadline: return response + if _should_read_error_body(response): + read_started = self._monotonic() + peek_until = min( + read_started + max(delay, PEEK_BUDGET_SECONDS), deadline + ) + peek = _Peek() + read_error: Optional[Exception] = None + try: + peek = await _aread_error_body( + response, peek_until, self._monotonic + ) + except (httpx.HTTPError, httpx.StreamError) as exc: + # A body that cannot be read decides nothing: retry as before. + read_error = exc + except BaseException: + # Interrupted mid-peek (e.g. cancelled): do not leak the + # connection. + await response.aclose() + raise + if peek.content is None and peek.rest is not None: + # Cut off on what was the last chunk: the buffered bytes are one + # complete JSON document, so they are the body read in full (a + # refusal or any other complete API error) and the tail is left + # unread; only a body that does not yet parse is handed on. + buffered = b"".join(peek.buffered) + if len(buffered) <= MAX_ERROR_BODY_BYTES and _is_complete_json( + buffered + ): + await _aclose_rest(peek) + peek = _Peek(content=buffered) + if peek.content is not None and _is_non_retryable_refusal(peek.content): + await response.aclose() + return _rebuild(response, request, content=peek.content) + # The time spent reading counts towards the delay. + delay = max(0.0, delay - (self._monotonic() - read_started)) + if self._monotonic() + delay >= deadline: + # Out of budget: the 503 is the outcome, as on the + # JavaScript side. A body read in full is returned as is; + # one stopped at the byte cap or cut off by the peek budget + # is handed on whole, the peeked chunks first and the rest + # of the stream after; a failed read is reported as such, + # never as a fabricated body (the API client parses every + # 503 body as JSON). + if peek.content is not None: + await response.aclose() + return _rebuild(response, request, content=peek.content) + if peek.rest is not None: + return _rebuild( + response, + request, + stream=_AsyncResumedStream( + response, peek, request, deadline, self._monotonic + ), + ) + await response.aclose() + if read_error is not None: + raise read_error + raise AssertionError("unreachable") + # Retrying: release what the peek left of the body. + await _aclose_rest(peek) await response.aclose() await self._sleep(delay) diff --git a/packages/python-sdk/e2b/sandbox/sandbox_api.py b/packages/python-sdk/e2b/sandbox/sandbox_api.py index 7b75c8d3f3..868eb2e026 100644 --- a/packages/python-sdk/e2b/sandbox/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox/sandbox_api.py @@ -573,6 +573,24 @@ def resolve_snapshot_memory( return mode == "full" +@dataclass +class SandboxPauseResult: + """ + Result of a pause. + """ + + paused: bool + """`True` if this call paused the sandbox, `False` if it was already paused.""" + + forced: bool + """ + Whether the snapshot was taken with the `force` override, without the guest + agent health check. Reports what happened rather than echoing the option: + `False` when the sandbox was already paused, and also when a forced request + joined a normal pause that was already in flight. + """ + + class SandboxOnTimeoutPause(TypedDict): """ Object form of `on_timeout` that auto-pauses the sandbox when the timeout is diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py index 3d0a648f50..a2723f9e56 100644 --- a/packages/python-sdk/e2b/sandbox_async/main.py +++ b/packages/python-sdk/e2b/sandbox_async/main.py @@ -31,6 +31,7 @@ SandboxNetworkUpdate, SandboxOnResume, SnapshotInfo, + SandboxPauseResult, SnapshotMode, ) from e2b.sandbox.utils import class_method_variant @@ -789,15 +790,19 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -808,16 +813,20 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox specified by sandbox ID. :param sandbox_id: Sandbox ID :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -827,21 +836,26 @@ async def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ return await SandboxApi._cls_pause( sandbox_id=self.sandbox_id, keep_memory=keep_memory, mode=mode, + force=force, **self.connection_config.get_api_params(**opts), ) @@ -851,8 +865,9 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @overload @staticmethod @@ -861,8 +876,9 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @class_method_variant("_cls_pause") async def beta_pause( @@ -870,14 +886,15 @@ async def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ :deprecated: Use `pause()` instead. - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) """ - return await self.pause(keep_memory=keep_memory, mode=mode, **opts) + return await self.pause(keep_memory=keep_memory, mode=mode, force=force, **opts) @overload async def create_snapshot( diff --git a/packages/python-sdk/e2b/sandbox_async/sandbox_api.py b/packages/python-sdk/e2b/sandbox_async/sandbox_api.py index e41f8ff397..d735c2dc2c 100644 --- a/packages/python-sdk/e2b/sandbox_async/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox_async/sandbox_api.py @@ -33,6 +33,7 @@ SandboxForkRequest, SandboxNetworkConfig, SandboxPauseRequest, + SandboxPauseResult as SandboxPauseResultAPI, SandboxVolumeMount as SandboxVolumeMountAPI, ) from e2b.api.client.types import UNSET, Unset @@ -56,6 +57,7 @@ SandboxNetworkOpts, SandboxNetworkUpdate, SandboxOnResume, + SandboxPauseResult, SnapshotMode, resolve_connect_memory, resolve_snapshot_memory, @@ -400,8 +402,9 @@ async def _cls_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: config = ConnectionConfig(**cls._resolve_api_params(**opts)) api_client = get_api_client(config) @@ -409,7 +412,8 @@ async def _cls_pause( sandbox_id, client=api_client, body=SandboxPauseRequest( - memory=resolve_snapshot_memory("mode", mode, keep_memory) + memory=resolve_snapshot_memory("mode", mode, keep_memory), + force=UNSET if force is None else force, ), ) @@ -418,7 +422,7 @@ async def _cls_pause( if res.status_code == 409: # Sandbox is already paused - return False + return SandboxPauseResult(paused=False, forced=False) if res.status_code >= 300: raise handle_api_exception(res) @@ -427,7 +431,13 @@ async def _cls_pause( if isinstance(res.parsed, Error): raise SandboxException(f"{res.parsed.message}: Request failed") - return True + # An older API answers 204 without a body. + forced = ( + res.parsed.forced + if isinstance(res.parsed, SandboxPauseResultAPI) + else False + ) + return SandboxPauseResult(paused=True, forced=forced) @classmethod async def _cls_fork( @@ -484,7 +494,15 @@ async def _cls_fork( # would have failed the whole request — so stay generic. exception = NotFoundException(f"{error.code}: {error.message}") else: - exception = api_exception_from_code(error.code, error.message) + exception = api_exception_from_code( + error.code, + error.message, + error_code=( + error.error_code + if isinstance(error.error_code, str) + else None + ), + ) results.append(exception) continue diff --git a/packages/python-sdk/e2b/sandbox_sync/main.py b/packages/python-sdk/e2b/sandbox_sync/main.py index 32c1b671b7..bfa50db984 100644 --- a/packages/python-sdk/e2b/sandbox_sync/main.py +++ b/packages/python-sdk/e2b/sandbox_sync/main.py @@ -30,6 +30,7 @@ SandboxNetworkUpdate, SandboxOnResume, SnapshotInfo, + SandboxPauseResult, SnapshotMode, ) from e2b.sandbox.utils import class_method_variant @@ -787,15 +788,19 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -806,16 +811,20 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox specified by sandbox ID. :param sandbox_id: Sandbox ID :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ ... @@ -825,21 +834,26 @@ def pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ Pause the sandbox. :param keep_memory: :deprecated: Use `mode` instead: `keep_memory=False` is `mode="filesystem"`, `keep_memory=True` is `mode="full"`. :param mode: What the pause persists. With `"filesystem"`, the in-memory state is dropped and only the filesystem is persisted; resuming such a sandbox cold-boots (reboots) it from disk, losing running processes and open connections. Defaults to `"full"` (memory and filesystem). + :param force: Take the pause even when the sandbox guest agent is unresponsive. A memory pause first checks that the guest agent answers; when it does not, the node refuses the pause (`SandboxPauseRefusedException`, whose message carries the reason) and the sandbox keeps running. With `force=True` the snapshot is taken without that check. Its memory may then not resume: if restoring it fails, resume with `on_resume="reboot"` to recover the disk state. A refusal for another reason is not lifted by `force`. Applies to memory pauses only; the API rejects it together with `mode="filesystem"`. Defaults to `False`. + + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :raises SandboxPauseRefusedException: if the node refused the pause; the message carries the reason """ return SandboxApi._cls_pause( sandbox_id=self.sandbox_id, keep_memory=keep_memory, mode=mode, + force=force, **self.connection_config.get_api_params(**opts), ) @@ -849,8 +863,9 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @overload @staticmethod @@ -859,8 +874,9 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: ... + ) -> SandboxPauseResult: ... @class_method_variant("_cls_pause") def beta_pause( @@ -868,14 +884,15 @@ def beta_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: """ :deprecated: Use `pause()` instead. - :return: `True` if the sandbox got paused, `False` if the sandbox was already paused + :return: The pause result: `paused` is `True` if this call paused the sandbox and `False` if it was already paused; `forced` whether the snapshot was taken without the guest agent health check (reported by the API, so a forced request that joined a normal pause already in flight comes back with `forced=False`) """ - return self.pause(keep_memory=keep_memory, mode=mode, **opts) + return self.pause(keep_memory=keep_memory, mode=mode, force=force, **opts) @overload def create_snapshot( diff --git a/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py b/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py index 8f4db3e890..82d30cc142 100644 --- a/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py +++ b/packages/python-sdk/e2b/sandbox_sync/sandbox_api.py @@ -33,6 +33,7 @@ SandboxForkRequest, SandboxNetworkConfig, SandboxPauseRequest, + SandboxPauseResult as SandboxPauseResultAPI, SandboxVolumeMount as SandboxVolumeMountAPI, ) from e2b.api.client.types import UNSET, Unset @@ -55,6 +56,7 @@ SandboxNetworkOpts, SandboxNetworkUpdate, SandboxOnResume, + SandboxPauseResult, SnapshotMode, resolve_connect_memory, resolve_snapshot_memory, @@ -413,7 +415,15 @@ def _cls_fork( # would have failed the whole request — so stay generic. exception = NotFoundException(f"{error.code}: {error.message}") else: - exception = api_exception_from_code(error.code, error.message) + exception = api_exception_from_code( + error.code, + error.message, + error_code=( + error.error_code + if isinstance(error.error_code, str) + else None + ), + ) results.append(exception) continue @@ -488,8 +498,9 @@ def _cls_pause( keep_memory: Optional[bool] = None, *, mode: Optional[SnapshotMode] = None, + force: Optional[bool] = None, **opts: Unpack[ApiParams], - ) -> bool: + ) -> SandboxPauseResult: config = ConnectionConfig(**cls._resolve_api_params(**opts)) api_client = get_api_client(config) @@ -497,7 +508,8 @@ def _cls_pause( sandbox_id, client=api_client, body=SandboxPauseRequest( - memory=resolve_snapshot_memory("mode", mode, keep_memory) + memory=resolve_snapshot_memory("mode", mode, keep_memory), + force=UNSET if force is None else force, ), ) @@ -506,7 +518,7 @@ def _cls_pause( if res.status_code == 409: # Sandbox is already paused - return False + return SandboxPauseResult(paused=False, forced=False) if res.status_code >= 300: raise handle_api_exception(res) @@ -515,4 +527,10 @@ def _cls_pause( if isinstance(res.parsed, Error): raise SandboxException(f"{res.parsed.message}: Request failed") - return True + # An older API answers 204 without a body. + forced = ( + res.parsed.forced + if isinstance(res.parsed, SandboxPauseResultAPI) + else False + ) + return SandboxPauseResult(paused=True, forced=forced) diff --git a/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py b/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py index 7a1ecfacda..66f1ee07b9 100644 --- a/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py +++ b/packages/python-sdk/tests/async/sandbox_async/test_snapshot_filesystem_only.py @@ -13,7 +13,7 @@ async def test_pause_filesystem_only(async_sandbox: AsyncSandbox): ).strip() # Filesystem-only pause: only the rootfs is persisted, no memory snapshot. - assert await async_sandbox.pause(mode="filesystem", request_timeout=120) + assert (await async_sandbox.pause(mode="filesystem", request_timeout=120)).paused assert not await async_sandbox.is_running() # Resuming a filesystem-only snapshot cold-boots (reboots) from the rootfs. diff --git a/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py b/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py index 6945e35f4e..78c2940514 100644 --- a/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py +++ b/packages/python-sdk/tests/shared/sandbox/test_api_defaults.py @@ -187,8 +187,12 @@ async def test_async_fork_rejects_count_outside_1_to_20( request.assert_not_called() +def _paused(): + return SimpleNamespace(status_code=200, parsed=SimpleNamespace(forced=False)) + + def _sync_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: - request = Mock(return_value=SimpleNamespace(status_code=204, parsed=None)) + request = Mock(return_value=_paused()) monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "sync_detailed", request) Sandbox.pause("sbx-test", api_key=api_key, **kwargs) @@ -197,7 +201,7 @@ def _sync_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: async def _async_pause_body(monkeypatch, api_key: str, **kwargs) -> Dict[str, Any]: - request = AsyncMock(return_value=SimpleNamespace(status_code=204, parsed=None)) + request = AsyncMock(return_value=_paused()) monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "asyncio_detailed", request) await AsyncSandbox.pause("sbx-test", api_key=api_key, **kwargs) @@ -227,6 +231,26 @@ def test_pause_still_sends_the_deprecated_keep_memory(monkeypatch, test_api_key) assert body["memory"] is False +def test_pause_omits_force_when_unset(monkeypatch, test_api_key): + body = _sync_pause_body(monkeypatch, test_api_key) + + assert "force" not in body + + +@pytest.mark.parametrize("force", [True, False]) +def test_pause_sends_an_explicit_force(monkeypatch, test_api_key, force): + body = _sync_pause_body(monkeypatch, test_api_key, force=force) + + assert body["force"] is force + + +def test_pause_sends_force_alongside_mode(monkeypatch, test_api_key): + body = _sync_pause_body(monkeypatch, test_api_key, mode="full", force=True) + + assert body["memory"] is True + assert body["force"] is True + + @pytest.mark.parametrize( "kwargs, match", [ @@ -265,6 +289,19 @@ async def test_async_pause_still_sends_the_deprecated_keep_memory( assert body["memory"] is False +async def test_async_pause_omits_force_when_unset(monkeypatch, test_api_key): + body = await _async_pause_body(monkeypatch, test_api_key) + + assert "force" not in body + + +@pytest.mark.parametrize("force", [True, False]) +async def test_async_pause_sends_an_explicit_force(monkeypatch, test_api_key, force): + body = await _async_pause_body(monkeypatch, test_api_key, force=force) + + assert body["force"] is force + + def _snapshot_response(): return SimpleNamespace( status_code=201, diff --git a/packages/python-sdk/tests/shared/sandbox/test_pause.py b/packages/python-sdk/tests/shared/sandbox/test_pause.py new file mode 100644 index 0000000000..a3e15ba28e --- /dev/null +++ b/packages/python-sdk/tests/shared/sandbox/test_pause.py @@ -0,0 +1,756 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import httpx + +import pytest + +from e2b import ( + AsyncSandbox, + TimeoutException, + Sandbox, + SandboxPauseResult, + SandboxException, + SandboxNotFoundException, + SandboxPauseRefusedException, + ServiceBusyException, +) +import e2b.sandbox_async.sandbox_api as async_sandbox_api +import e2b.sandbox_sync.sandbox_api as sync_sandbox_api +from e2b.api import ApiClient, AsyncApiClient +from e2b.retry import AsyncRetryableTransport, RetryableTransport +from e2b.api.client.api.sandboxes import post_sandboxes_sandbox_id_pause +from e2b.api.client.models import Error +from e2b.api.client.models import SandboxPauseResult as SandboxPauseResultAPI + +REFUSED_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because the sandbox guest " + "agent is not responding: retry, or pause with force to take the snapshot " + "without the health check" +) +BUSY_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because its node is busy, " + "please retry" +) +DRAINING_MESSAGE = ( + "Sandbox 'sbx-test' cannot be paused right now because the server is " + "shutting down, please retry" +) + + +def _response(status_code: int, body: dict): + """A pause response the way the generated client builds it: the raw body + and, for an error status, the parsed ``Error`` model.""" + parsed = ( + SandboxPauseResultAPI.from_dict(body) + if status_code == 200 + else Error.from_dict(body) + ) + return SimpleNamespace( + status_code=status_code, + content=json.dumps(body).encode(), + parsed=parsed, + headers={}, + ) + + +def _answer(monkeypatch, response): + request = Mock(return_value=response) + monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "sync_detailed", request) + return request + + +def _answer_async(monkeypatch, response): + request = AsyncMock(return_value=response) + monkeypatch.setattr(post_sandboxes_sandbox_id_pause, "asyncio_detailed", request) + return request + + +def test_pause_returns_paused_not_forced(monkeypatch, test_api_key): + _answer(monkeypatch, _response(200, {"forced": False})) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=True, forced=False + ) + + +def test_forced_pause_returns_paused_and_forced(monkeypatch, test_api_key): + request = _answer(monkeypatch, _response(200, {"forced": True})) + + assert Sandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=True) + assert request.call_args.kwargs["body"].to_dict()["force"] is True + + +def test_forced_reports_what_the_api_did_not_the_option(monkeypatch, test_api_key): + # A forced request that joined a normal pause already in flight is not forced. + _answer(monkeypatch, _response(200, {"forced": False})) + + assert Sandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +def test_pause_on_an_older_api_answering_204_is_paused_not_forced( + monkeypatch, test_api_key +): + _answer(monkeypatch, SimpleNamespace(status_code=204, parsed=None, content=b"")) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=True, forced=False + ) + + +def test_pause_of_an_already_paused_sandbox_is_not_paused(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response(409, {"code": 409, "message": "Sandbox is already paused"}), + ) + + assert Sandbox.pause("sbx-test", api_key=test_api_key) == SandboxPauseResult( + paused=False, forced=False + ) + + +def test_refused_pause_raises_the_typed_exception(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + err = excinfo.value + # Subclass of the generic 503 exception, so handlers written for it keep working. + assert isinstance(err, ServiceBusyException) + assert err.status_code == 503 + assert err.error_code == "sandbox_pause_refused" + # The API's message verbatim: it names the reason and what to do. + assert str(err) == REFUSED_MESSAGE + + +def test_refusal_for_a_busy_node_carries_the_same_code_and_the_reason( + monkeypatch, test_api_key +): + _answer( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": BUSY_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == BUSY_MESSAGE + + +def test_503_with_no_error_code_at_all_stays_a_busy_exception( + monkeypatch, test_api_key +): + _answer(monkeypatch, _response(503, {"code": 503, "message": DRAINING_MESSAGE})) + + with pytest.raises(ServiceBusyException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "server is shutting down" in str(excinfo.value) + + +def test_force_on_a_filesystem_only_pause_is_a_400(monkeypatch, test_api_key): + _answer( + monkeypatch, + _response(400, {"code": 400, "message": "force applies to memory pauses only"}), + ) + + with pytest.raises(SandboxException) as excinfo: + Sandbox.pause("sbx-test", mode="filesystem", force=True, api_key=test_api_key) + + assert excinfo.value.status_code == 400 + assert "force applies to memory pauses only" in str(excinfo.value) + + +def test_pause_of_a_missing_sandbox_raises_not_found(monkeypatch, test_api_key): + _answer(monkeypatch, _response(404, {"code": 404, "message": "Sandbox not found"})) + + with pytest.raises(SandboxNotFoundException): + Sandbox.pause("sbx-test", api_key=test_api_key) + + +async def test_async_pause_returns_paused_not_forced(monkeypatch, test_api_key): + _answer_async(monkeypatch, _response(200, {"forced": False})) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +async def test_async_forced_pause_returns_paused_and_forced(monkeypatch, test_api_key): + request = _answer_async(monkeypatch, _response(200, {"forced": True})) + + assert await AsyncSandbox.pause( + "sbx-test", force=True, api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=True) + assert request.call_args.kwargs["body"].to_dict()["force"] is True + + +async def test_async_pause_on_an_older_api_answering_204_is_paused_not_forced( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, SimpleNamespace(status_code=204, parsed=None, content=b"") + ) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=True, forced=False) + + +async def test_async_pause_of_an_already_paused_sandbox_is_not_paused( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response(409, {"code": 409, "message": "Sandbox is already paused"}), + ) + + assert await AsyncSandbox.pause( + "sbx-test", api_key=test_api_key + ) == SandboxPauseResult(paused=False, forced=False) + + +async def test_async_refused_pause_raises_the_typed_exception( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response( + 503, + { + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert isinstance(excinfo.value, ServiceBusyException) + assert excinfo.value.status_code == 503 + assert str(excinfo.value) == REFUSED_MESSAGE + + +async def test_async_503_with_no_error_code_at_all_stays_a_busy_exception( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, _response(503, {"code": 503, "message": DRAINING_MESSAGE}) + ) + + with pytest.raises(ServiceBusyException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + + +async def test_async_force_on_a_filesystem_only_pause_is_a_400( + monkeypatch, test_api_key +): + _answer_async( + monkeypatch, + _response(400, {"code": 400, "message": "force applies to memory pauses only"}), + ) + + with pytest.raises(SandboxException) as excinfo: + await AsyncSandbox.pause( + "sbx-test", mode="filesystem", force=True, api_key=test_api_key + ) + + assert excinfo.value.status_code == 400 + + +# The tests above mock the generated client's `sync_detailed`; these go through +# its real response parsing over an httpx mock transport, so the 200 body is +# parsed into the generated model and the 204 path sees `parsed is None`. + + +def _mock_api(monkeypatch, answer, clock=None): + """Route both clients through the SDK's own retry transports over an + httpx mock transport whose responses, like pyqwest's, carry no request + (httpx attaches it after the transport returns). ``answer`` builds a fresh + response per request; ``clock`` replaces the transports' monotonic clock + and sleep.""" + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return answer() + + def sync_client(config): + inner = httpx.MockTransport(handler) + if clock is None: + transport = RetryableTransport(inner, config.retries) + else: + transport = RetryableTransport( + inner, config.retries, sleep=clock.sleep, monotonic=clock.monotonic + ) + return ApiClient(config, transport=transport) + + def async_client(config): + inner = httpx.MockTransport(handler) + if clock is None: + transport = AsyncRetryableTransport(inner, config.retries) + else: + fake = clock + + async def async_sleep(delay: float) -> None: + fake.sleep(delay) + + transport = AsyncRetryableTransport( + inner, config.retries, sleep=async_sleep, monotonic=fake.monotonic + ) + return AsyncApiClient(config, transport=transport) + + monkeypatch.setattr(sync_sandbox_api, "get_api_client", sync_client) + monkeypatch.setattr(async_sandbox_api, "get_api_client", async_client) + return seen + + +@pytest.mark.parametrize("forced", [True, False]) +def test_pause_parses_the_200_body(monkeypatch, test_api_key, forced): + seen = _mock_api(monkeypatch, lambda: httpx.Response(200, json={"forced": forced})) + + result = Sandbox.pause("sbx-test", force=forced, api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=forced) + assert json.loads(seen[0].content) == {"force": forced} + + +def test_pause_accepts_a_204_from_an_older_api(monkeypatch, test_api_key): + seen = _mock_api(monkeypatch, lambda: httpx.Response(204)) + + result = Sandbox.pause("sbx-test", api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=False) + assert json.loads(seen[0].content) == {} + + +@pytest.mark.parametrize("forced", [True, False]) +async def test_async_pause_parses_the_200_body(monkeypatch, test_api_key, forced): + seen = _mock_api(monkeypatch, lambda: httpx.Response(200, json={"forced": forced})) + + result = await AsyncSandbox.pause("sbx-test", force=forced, api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=forced) + assert json.loads(seen[0].content) == {"force": forced} + + +async def test_async_pause_accepts_a_204_from_an_older_api(monkeypatch, test_api_key): + _mock_api(monkeypatch, lambda: httpx.Response(204)) + + result = await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert result == SandboxPauseResult(paused=True, forced=False) + + +def test_pause_refusal_through_the_real_parsing(monkeypatch, test_api_key): + _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + json={ + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == REFUSED_MESSAGE + + +async def test_async_pause_refusal_through_the_real_parsing(monkeypatch, test_api_key): + _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + json={ + "code": 503, + "error_code": "sandbox_pause_refused", + "message": REFUSED_MESSAGE, + }, + ), + ) + + with pytest.raises(SandboxPauseRefusedException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key) + + assert str(excinfo.value) == REFUSED_MESSAGE + + +class _Clock: + def __init__(self): + self.now = 0.0 + self.sleeps = [] + + def monotonic(self): + return self.now + + def sleep(self, delay): + self.sleeps.append(delay) + self.now += delay + + +class _StalledBody(httpx.SyncByteStream): + """A 503 body whose read eats 2 s of the clock and then times out.""" + + def __init__(self, clock): + self.clock = clock + + def __iter__(self): + self.clock.now += 2.0 + raise httpx.ReadTimeout("stalled") + yield b"" # pragma: no cover + + +def test_pause_whose_refusal_body_read_times_out_with_no_budget_left_reports_the_timeout( + monkeypatch, test_api_key +): + clock = _Clock() + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=_StalledBody(clock), + headers={"Content-Type": "application/json", "Content-Length": "40"}, + ), + clock=clock, + ) + + # A 1 s request timeout is the whole retry budget; the read outlives it. + with pytest.raises(Exception) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert isinstance(excinfo.value, (httpx.TimeoutException, TimeoutException)) + assert len(seen) == 1 + + +REFUSAL_CHUNKS = ( + b'{"code":503,"error_code":"sand', + b"box_pause_", + b'refused","message":"refused"}', +) + + +class _DripBody(httpx.SyncByteStream): + """A 503 body whose every chunk eats 0.6 s of the clock; ``tail`` chunks + follow the complete refusal and must never be pulled. Counts closes.""" + + def __init__(self, clock, chunks=REFUSAL_CHUNKS, tail=()): + self.clock = clock + self.chunks = chunks + self.tail = tail + self.closes = 0 + self.pulled = 0 + + def __iter__(self): + for chunk in self.chunks: + self.clock.now += 0.6 + self.pulled += 1 + yield chunk + for chunk in self.tail: + self.pulled += 1 + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncDripBody(httpx.AsyncByteStream): + def __init__(self, clock, chunks=REFUSAL_CHUNKS, tail=()): + self.clock = clock + self.chunks = chunks + self.tail = tail + self.closes = 0 + self.pulled = 0 + + async def __aiter__(self): + for chunk in self.chunks: + self.clock.now += 0.6 + self.pulled += 1 + yield chunk + for chunk in self.tail: + self.pulled += 1 + yield chunk + + async def aclose(self): + self.closes += 1 + + +def test_pause_whose_refusal_body_trickles_past_the_budget_still_surfaces_the_refusal( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + # Three chunks at 0.6 s against the 1 s request timeout: the peek is cut + # off on the third chunk, which completes the refusal, so the buffered + # bytes are used as the body and the live tail is never read. + with pytest.raises(SandboxPauseRefusedException): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +class _BigRefusalBody(httpx.SyncByteStream): + """A refusal body past the 8 KiB cap, in three chunks of 0.5 s each, so + only the cap stops the peek. Counts closes and pulls.""" + + BODY = ( + b'{"code":503,"error_code":"sandbox_pause_refused","message":"' + + b"x" * 9000 + + b'"}' + ) + + def __init__(self, clock): + self.clock = clock + self.closes = 0 + self.pulled = 0 + + def __iter__(self): + for chunk in (self.BODY[:4200], self.BODY[4200:8400], self.BODY[8400:]): + self.clock.now += 0.5 + self.pulled += 1 + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncBigRefusalBody(httpx.AsyncByteStream): + def __init__(self, clock): + self.clock = clock + self.closes = 0 + self.pulled = 0 + + async def __aiter__(self): + body = _BigRefusalBody.BODY + for chunk in (body[:4200], body[4200:8400], body[8400:]): + self.clock.now += 0.5 + self.pulled += 1 + yield chunk + + async def aclose(self): + self.closes += 1 + + +def test_pause_whose_refusal_body_passes_the_cap_with_no_budget_left_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _BigRefusalBody(clock) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=body, + headers={ + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + }, + ), + clock=clock, + ) + + # Over the cap on the second chunk at the 1 s request timeout: the 503 is + # handed on, but its third chunk would be waited for past the deadline, + # so the read ends there with a timeout instead of running on; the inner + # response is closed once. + with pytest.raises(httpx.ReadTimeout): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 2 + assert body.closes == 1 + + +# The refusal split so that the third chunk leaves the JSON open: the peek cut +# off there holds no refusal yet, and the body is handed on with a live tail. +OPEN_REFUSAL_CHUNKS = ( + b'{"code":503,"error_code":"sand', + b"box_pause_", + b'refused","message":"refused"', +) + + +def test_pause_whose_handed_on_body_trickles_past_the_deadline_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, chunks=OPEN_REFUSAL_CHUNKS, tail=(b"}",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + # Cut off on the third chunk with the JSON still open and no budget left: + # the 503 is handed on, but its tail would be read past the 1 s request + # timeout, so the read ends there with a timeout instead of running on. + with pytest.raises(httpx.ReadTimeout): + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_handed_on_body_trickles_past_the_deadline_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, chunks=OPEN_REFUSAL_CHUNKS, tail=(b"}",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(httpx.ReadTimeout): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_refusal_body_passes_the_cap_with_no_budget_left_times_out_at_the_deadline( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncBigRefusalBody(clock) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, + stream=body, + headers={ + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + }, + ), + clock=clock, + ) + + with pytest.raises(httpx.ReadTimeout): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 2 + assert body.closes == 1 + + +async def test_async_pause_whose_refusal_body_trickles_past_the_budget_still_surfaces_the_refusal( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(SandboxPauseRefusedException): + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +# A busy 503 (no refusal code) whose complete body lands in the very chunk that +# stops the peek: it must surface as the API error it is, not as a timeout. +BUSY_CHUNKS = (b'{"code":503,"message":"node is bu', b"sy persisting ", b'sandbox"}') + + +def test_pause_whose_busy_body_completes_in_the_cut_off_chunk_surfaces_the_busy_error( + monkeypatch, test_api_key +): + clock = _Clock() + body = _DripBody(clock, chunks=BUSY_CHUNKS, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(ServiceBusyException) as excinfo: + Sandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "node is busy persisting sandbox" in str(excinfo.value) + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 + + +async def test_async_pause_whose_busy_body_completes_in_the_cut_off_chunk_surfaces_the_busy_error( + monkeypatch, test_api_key +): + clock = _Clock() + body = _AsyncDripBody(clock, chunks=BUSY_CHUNKS, tail=(b"\n",)) + seen = _mock_api( + monkeypatch, + lambda: httpx.Response( + 503, stream=body, headers={"Content-Type": "application/json"} + ), + clock=clock, + ) + + with pytest.raises(ServiceBusyException) as excinfo: + await AsyncSandbox.pause("sbx-test", api_key=test_api_key, request_timeout=1) + + assert not isinstance(excinfo.value, SandboxPauseRefusedException) + assert "node is busy persisting sandbox" in str(excinfo.value) + assert len(seen) == 1 + assert body.pulled == 3 + assert body.closes == 1 diff --git a/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py b/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py index dd688cbf7b..a51feac2ae 100644 --- a/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py +++ b/packages/python-sdk/tests/sync/sandbox_sync/test_snapshot_filesystem_only.py @@ -11,7 +11,7 @@ def test_pause_filesystem_only(sandbox: Sandbox): boot_before = sandbox.files.read("/proc/sys/kernel/random/boot_id").strip() # Filesystem-only pause: only the rootfs is persisted, no memory snapshot. - assert sandbox.pause(mode="filesystem", request_timeout=120) + assert sandbox.pause(mode="filesystem", request_timeout=120).paused assert not sandbox.is_running() # Resuming a filesystem-only snapshot cold-boots (reboots) from the rootfs. diff --git a/packages/python-sdk/tests/test_api_exception.py b/packages/python-sdk/tests/test_api_exception.py index cd6dc81686..c32d81a1b2 100644 --- a/packages/python-sdk/tests/test_api_exception.py +++ b/packages/python-sdk/tests/test_api_exception.py @@ -3,6 +3,7 @@ from e2b.api import api_exception_from_code, handle_api_exception from e2b.exceptions import ( RateLimitException, + SandboxPauseRefusedException, ServiceBusyException, SandboxException, ) @@ -22,6 +23,43 @@ def test_refused_pause_is_a_busy_exception_with_the_status_and_message(): assert "node is busy persisting sandbox" in str(err) +def test_503_with_the_refused_code_is_a_pause_refused_exception_whatever_the_endpoint(): + reason = "Sandbox 'sbx' cannot be paused right now because its node is busy, please retry" + err = handle_api_exception( + response( + 503, + ( + '{"code":503,"error_code":"sandbox_pause_refused","message":"' + + reason + + '"}' + ).encode(), + ) + ) + + assert isinstance(err, SandboxPauseRefusedException) + assert isinstance(err, ServiceBusyException) + assert err.status_code == 503 + assert err.error_code == "sandbox_pause_refused" + assert str(err) == reason + + +def test_503_with_another_code_stays_a_busy_exception(): + err = api_exception_from_code( + 503, "no capacity", error_code="sandbox_capacity_unavailable" + ) + + assert isinstance(err, ServiceBusyException) + assert not isinstance(err, SandboxPauseRefusedException) + + +def test_503_with_a_json_list_body_is_still_a_busy_exception(): + err = handle_api_exception(response(503, b'["not", "an", "object"]')) + + assert isinstance(err, ServiceBusyException) + assert not isinstance(err, SandboxPauseRefusedException) + assert err.status_code == 503 + + def test_503_without_a_body_is_still_a_busy_exception(): err = handle_api_exception(response(503)) diff --git a/packages/python-sdk/tests/test_rate_limit_retry_transport.py b/packages/python-sdk/tests/test_rate_limit_retry_transport.py index c4699a8b72..3316d5c05f 100644 --- a/packages/python-sdk/tests/test_rate_limit_retry_transport.py +++ b/packages/python-sdk/tests/test_rate_limit_retry_transport.py @@ -1,13 +1,21 @@ +import asyncio +import threading + import httpx import pytest from e2b.retry import ( + _AsyncResumedStream, + _Peek, + _ResumedStream, AsyncRetryableTransport, RetryableTransport, is_replayable, parse_retry_after, ) +JSON = {"Content-Type": "application/json"} + class FakeTransport(httpx.BaseTransport): def __init__(self, outcomes, retry_after="0"): @@ -26,7 +34,6 @@ def handle_request(self, request): outcome, headers={"Retry-After": self.retry_after} if self.retry_after else {}, content=b"response", - request=request, ) self.responses.append(response) return response @@ -52,7 +59,6 @@ async def handle_async_request(self, request): outcome, headers={"Retry-After": self.retry_after} if self.retry_after else {}, content=b"response", - request=request, ) self.responses.append(response) return response @@ -218,6 +224,1204 @@ def test_bad_gateway_is_retried_only_for_replayable_operations(method, path, att assert len(inner.requests) == attempts +class JsonTransport(httpx.BaseTransport): + """Answers each request with the next (status, body) pair.""" + + def __init__(self, outcomes): + self.outcomes = iter(outcomes) + self.requests = [] + + def handle_request(self, request): + self.requests.append(request) + status, body = next(self.outcomes) + return httpx.Response(status, json=body) + + +class JsonAsyncTransport(httpx.AsyncBaseTransport): + def __init__(self, outcomes): + self.outcomes = iter(outcomes) + self.requests = [] + + async def handle_async_request(self, request): + self.requests.append(request) + status, body = next(self.outcomes) + return httpx.Response(status, json=body) + + +REFUSED = {"code": 503, "error_code": "sandbox_pause_refused", "message": "refused"} +CAPACITY = {"code": 503, "error_code": "sandbox_capacity_unavailable", "message": "no"} + + +def test_service_unavailable_with_a_non_retryable_error_code_is_returned_after_one_request(): + inner = JsonTransport([(503, REFUSED)]) + sleeps = [] + + response = RetryableTransport( + inner, retries=3, sleep=sleeps.append, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert sleeps == [] + + +def test_service_unavailable_with_an_oversized_body_is_retried_without_reading_it(): + big = {"code": 503, "error_code": "sandbox_pause_refused", "message": "x" * 9000} + inner = JsonTransport([(503, big), (200, {})]) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +class _ChunkedStream(httpx.SyncByteStream): + """A 503 body of unknown length that records whether it was read.""" + + def __init__(self, chunks, raise_=None): + self.chunks = chunks + self.raise_ = raise_ + self.read = False + + def __iter__(self): + self.read = True + if self.raise_ is not None: + raise self.raise_ + yield from self.chunks + + +class _AsyncChunkedStream(httpx.AsyncByteStream): + def __init__(self, chunks, raise_=None): + self.chunks = chunks + self.raise_ = raise_ + self.read = False + + async def __aiter__(self): + self.read = True + if self.raise_ is not None: + raise self.raise_ + for chunk in self.chunks: + yield chunk + + +class StreamTransport(httpx.BaseTransport): + """Answers each request with the next prepared response factory.""" + + def __init__(self, factories): + self.factories = iter(factories) + self.requests = [] + + def handle_request(self, request): + self.requests.append(request) + return next(self.factories)(request) + + +class AsyncStreamTransport(httpx.AsyncBaseTransport): + def __init__(self, factories): + self.factories = iter(factories) + self.requests = [] + + async def handle_async_request(self, request): + self.requests.append(request) + return next(self.factories)(request) + + +def test_chunked_refusal_without_content_length_is_returned_after_one_request(): + body = _ChunkedStream([b'{"error_code":"sandbox_', b'pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + # The body read for the peek is still the caller's to read. + assert response.json()["error_code"] == "sandbox_pause_refused" + + +async def test_async_chunked_refusal_without_content_length_is_returned_after_one_request(): + body = _AsyncChunkedStream([b'{"error_code":"sandbox_', b'pause_refused"}']) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + assert response.json()["error_code"] == "sandbox_pause_refused" + + +def test_oversized_chunked_body_is_retried(): + body = _ChunkedStream( + [b'{"error_code":"sandbox_pause_refused","pad":"' + b"x" * 9000 + b'"}'] + ) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_non_json_503_body_is_retried_without_reading_it(): + body = _ChunkedStream([b"busy"]) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=body, headers={"Content-Type": "text/html"} + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is False + + +def test_content_type_is_matched_by_media_type_whatever_its_case_and_parameters(): + body = _ChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Type": "Application/JSON; charset=UTF-8"}, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + + +def test_service_unavailable_whose_body_read_fails_is_retried(): + body = _ChunkedStream([], raise_=httpx.ReadTimeout("stalled")) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "40", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is True + + +async def test_async_service_unavailable_whose_body_read_fails_is_retried(): + body = _AsyncChunkedStream([], raise_=httpx.ReadTimeout("stalled")) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "40", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.read is True + + +async def test_async_service_unavailable_with_a_refusal_body_of_known_length_is_returned(): + body = _AsyncChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={"Content-Length": "38", "Content-Type": "application/json"}, + ), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert len(inner.requests) == 1 + + +class _SlowStream(httpx.SyncByteStream): + """A body whose read advances the fake clock, then yields or raises.""" + + def __init__(self, clock, seconds, chunks, raise_=None): + self.clock = clock + self.seconds = seconds + self.chunks = chunks + self.raise_ = raise_ + + def __iter__(self): + self.clock.now += self.seconds + if self.raise_ is not None: + raise self.raise_ + yield from self.chunks + + +class _AsyncSlowStream(httpx.AsyncByteStream): + def __init__(self, clock, seconds, chunks, raise_=None): + self.clock = clock + self.seconds = seconds + self.chunks = chunks + self.raise_ = raise_ + + async def __aiter__(self): + self.clock.now += self.seconds + if self.raise_ is not None: + raise self.raise_ + for chunk in self.chunks: + yield chunk + + +BUDGET_1S = {"timeout": {"connect": None, "read": 1.0, "write": None, "pool": None}} + + +def test_failed_503_body_read_with_no_budget_left_reraises_the_read_error(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 2.0, [], raise_=httpx.ReadTimeout("stalled")), + headers={"Content-Length": "40", **JSON}, + ), + ] + ) + + with pytest.raises(httpx.ReadTimeout, match="stalled"): + RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # The read outlived the 1 s budget: no retry, and the failed read is the + # truthful outcome (no fabricated empty body for the API client to parse). + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_failed_503_body_read_with_no_budget_left_reraises_the_read_error(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_AsyncSlowStream( + clock, 2.0, [], raise_=httpx.ReadTimeout("stalled") + ), + headers={"Content-Length": "40", **JSON}, + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + with pytest.raises(httpx.ReadTimeout, match="stalled"): + await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +def test_complete_503_body_read_with_no_budget_left_returns_it_with_the_body(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 2.0, [b'{"code":503,"message":"busy"}']), + headers={"Content-Length": "29", **JSON}, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + + +# A refusal in three chunks; the third completes the JSON. +DRIP = [b'{"error_code":"sand', b"box_pause_", b'refused"}'] +# The same refusal split so that the third chunk leaves the JSON open: a peek +# cut off there holds a prefix that does not parse, and hands the body on with +# a live tail. +OPEN_DRIP = [b'{"error_code":"sand', b"box_pause_", b'refused"', b"}"] + + +class _DripStream(httpx.SyncByteStream): + """A body whose every chunk advances the fake clock; counts closes.""" + + def __init__(self, clock, per_chunk, chunks): + self.clock = clock + self.per_chunk = per_chunk + self.chunks = chunks + self.closes = 0 + + def __iter__(self): + for chunk in self.chunks: + self.clock.now += self.per_chunk + yield chunk + + def close(self): + self.closes += 1 + + +class _AsyncDripStream(httpx.AsyncByteStream): + def __init__(self, clock, per_chunk, chunks): + self.clock = clock + self.per_chunk = per_chunk + self.chunks = chunks + self.closes = 0 + + async def __aiter__(self): + for chunk in self.chunks: + self.clock.now += self.per_chunk + yield chunk + + async def aclose(self): + self.closes += 1 + + +async def test_async_peek_stops_after_its_wall_clock_budget_and_the_503_is_retried(): + clock = FakeClock() + # The cut-off falls on the third chunk, which leaves the JSON open, so + # the buffered prefix is not yet a refusal. + chunks = OPEN_DRIP + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, chunks), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, + retries=3, + sleep=sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # 1.2 s into a 1 s peek budget the read stops with the JSON still open: + # not a refusal, retried. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_peek_stops_after_its_wall_clock_budget_and_the_503_is_retried(): + clock = FakeClock() + # The cut-off falls on the third chunk, which leaves the JSON open, so + # the buffered prefix is not yet a refusal. + chunks = OPEN_DRIP + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, chunks), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # 1.2 s into a 1 s peek budget the read stops with the JSON still open: + # not a refusal, retried. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_read_cut_off_by_the_budget_with_no_budget_left_hands_the_503_on_whole(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, DRIP), headers=JSON + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # Three chunks at 0.6 s against a 1 s budget: the third wait began past + # it and the request budget is gone too, so the 503 is the outcome and its + # body is handed on whole (peeked chunks, then the rest of the stream), as + # the JavaScript side does. + assert response.status_code == 503 + assert response.read() == b"".join(DRIP) + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_read_cut_off_by_the_budget_with_no_budget_left_hands_the_503_on_whole(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, DRIP), headers=JSON + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert (await response.aread()) == b"".join(DRIP) + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_complete_503_body_read_with_no_budget_left_returns_it_with_the_body(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_AsyncSlowStream(clock, 2.0, [b'{"code":503,"message":"busy"}']), + headers={"Content-Length": "29", **JSON}, + ), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + + +BIG_REFUSAL = ( + b'{"code":503,"error_code":"sandbox_pause_refused","message":"' + + b"x" * 9000 + + b'"}' +) +# The cap (8 KiB) is passed on the second chunk; the third is only ever read by +# the resumed stream's live-iterator replay. +BIG_CHUNKS = [BIG_REFUSAL[:4200], BIG_REFUSAL[4200:8400], BIG_REFUSAL[8400:]] +STREAMED_JSON = {**JSON, "Transfer-Encoding": "chunked"} + + +def test_over_cap_body_with_no_budget_left_is_handed_on_up_to_the_deadline(): + clock = FakeClock() + body = _DripStream(clock, 0.5, BIG_CHUNKS) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=STREAMED_JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + # Past the byte cap on the second chunk (0.5 s per chunk keeps every wait + # within the 1 s peek budget, so only the cap stops the peek). The peek's + # time counts against the retry delay, so a budget exhausted after it + # means the clock is at the request deadline: the 503 is handed on, the + # stream-form rebuild carries neither a stale length nor the transfer + # encoding, the peeked chunks are replayed, and the tail is not waited + # for past the deadline. + assert response.status_code == 503 + assert "content-length" not in response.headers + assert "transfer-encoding" not in response.headers + chunks = response.iter_bytes() + assert [next(chunks), next(chunks)] == BIG_CHUNKS[:2] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + next(chunks) + assert body.closes == 1 + assert len(inner.requests) == 1 + + +async def test_async_over_cap_body_with_no_budget_left_is_handed_on_up_to_the_deadline(): + clock = FakeClock() + body = _AsyncDripStream(clock, 0.5, BIG_CHUNKS) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=STREAMED_JSON)] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert "content-length" not in response.headers + assert "transfer-encoding" not in response.headers + received = [] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + async for chunk in response.aiter_bytes(): + received.append(chunk) + assert received == BIG_CHUNKS[:2] + assert body.closes == 1 + assert len(inner.requests) == 1 + + +DEEP = b"[" * 3000 + + +def test_a_503_body_nested_too_deep_to_parse_is_not_a_refusal_and_is_retried(): + inner = StreamTransport( + [ + lambda request: httpx.Response(503, content=DEEP, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # json.loads raises RecursionError here, not ValueError; the peek must + # swallow it like any unparsable body. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_a_503_body_nested_too_deep_to_parse_is_not_a_refusal_and_is_retried(): + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, content=DEEP, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +class _GatedStream(httpx.SyncByteStream): + """Sync twin of ``_GatedAsyncStream`` on threading events.""" + + def __init__(self, chunks, gate_before, clock=None, per_chunk=0.0): + self.chunks = chunks + self.gate_before = gate_before + self.clock = clock + self.per_chunk = per_chunk + self.gate = threading.Event() + self.waiting = threading.Event() + self.closes = 0 + + def __iter__(self): + for index, chunk in enumerate(self.chunks): + if index == self.gate_before: + self.waiting.set() + self.gate.wait() + if self.clock is not None: + self.clock.now += self.per_chunk + yield chunk + + def close(self): + self.closes += 1 + + +def test_closing_the_resumed_response_while_a_reader_thread_iterates_closes_the_inner_once(): + # Built directly with the deadline ahead: through the transport a hand-on + # happens at the deadline, where the tail is never waited for. + body = _GatedStream(OPEN_DRIP, gate_before=3) + inner = httpx.Response(503, stream=body, headers=JSON) + rest = inner.iter_bytes() + peek = _Peek(buffered=[next(rest)], rest=rest) + request = httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause") + response = httpx.Response( + 503, + stream=_ResumedStream(inner, peek, request, float("inf"), lambda: 0.0), + request=request, + ) + + received = [] + reader = threading.Thread( + target=lambda: received.extend(response.iter_bytes()), daemon=True + ) + reader.start() + assert body.waiting.wait(5) + + # The reader is inside the live generator (awaiting the fourth chunk); + # closing it from here raises ValueError("generator already executing"), + # which close() tolerates while still closing the inner exactly once. + response.close() + assert body.closes == 1 + + body.gate.set() + reader.join(5) + assert not reader.is_alive() + assert b"".join(received) == b"".join(OPEN_DRIP) + assert body.closes == 1 + + +UNHASHABLE_CODE = { + "code": 503, + "error_code": ["sandbox_pause_refused"], + "message": "a proxy rewrote the body", +} + + +def test_a_503_whose_error_code_is_not_a_string_is_not_a_refusal_and_is_retried(): + inner = StreamTransport( + [ + lambda request: httpx.Response(503, json=UNHASHABLE_CODE), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # A list is unhashable: without the string guard the membership test + # raised TypeError out of the transport. + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_a_503_whose_error_code_is_not_a_string_is_not_a_refusal_and_is_retried(): + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response(503, json=UNHASHABLE_CODE), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(_): + return None + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +def test_a_cut_off_peek_whose_buffered_chunks_form_the_refusal_returns_it_without_retrying(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_DripStream(clock, 0.6, DRIP), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # The third chunk's wait began past the 1 s budget, but it was the last + # one: the buffered bytes are the whole refusal, so no retry is needed + # although budget remains. + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +async def test_async_a_cut_off_peek_whose_buffered_chunks_form_the_refusal_returns_it_without_retrying(): + clock = FakeClock() + inner = AsyncStreamTransport( + [ + lambda request: httpx.Response( + 503, stream=_AsyncDripStream(clock, 0.6, DRIP), headers=JSON + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, + retries=3, + sleep=sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert clock.sleeps == [] + + +class _GatedAsyncStream(httpx.AsyncByteStream): + """Yields ``chunks`` advancing the clock per chunk, but waits on ``gate`` + before the chunk at ``gate_before``; counts closes.""" + + def __init__(self, chunks, gate_before, clock=None, per_chunk=0.0): + self.chunks = chunks + self.gate_before = gate_before + self.clock = clock + self.per_chunk = per_chunk + self.gate = asyncio.Event() + self.waiting = asyncio.Event() + self.closes = 0 + + async def __aiter__(self): + for index, chunk in enumerate(self.chunks): + if index == self.gate_before: + self.waiting.set() + await self.gate.wait() + if self.clock is not None: + self.clock.now += self.per_chunk + yield chunk + + async def aclose(self): + self.closes += 1 + + +async def test_async_cancellation_during_the_peek_closes_the_inner_response_once(): + body = _GatedAsyncStream( + [b'{"code":503,"error_code":"sand', b'box_pause_refused","message":"r"}'], + gate_before=1, + ) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + async def sleep(_): + return None + + task = asyncio.create_task( + AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request( + "POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}" + ) + ) + ) + await body.waiting.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + # The peek was awaiting the second chunk; the inner response is closed + # exactly once and the cancellation propagates. + assert body.closes == 1 + + +async def test_async_closing_the_resumed_response_while_a_reader_awaits_closes_the_inner_once(): + body = _GatedAsyncStream(OPEN_DRIP, gate_before=3) + inner = httpx.Response(503, stream=body, headers=JSON) + rest = inner.aiter_bytes() + peek = _Peek(buffered=[await rest.__anext__()], rest=rest) + request = httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause") + response = httpx.Response( + 503, + stream=_AsyncResumedStream(inner, peek, request, float("inf"), lambda: 0.0), + request=request, + ) + + received = [] + + async def read(): + async for chunk in response.aiter_bytes(): + received.append(chunk) + + reader = asyncio.create_task(read()) + await body.waiting.wait() + + # A second task closes while the reader awaits the fourth chunk inside the + # running generator: that generator cannot be closed from outside, and the + # inner response is still closed, exactly once. + await response.aclose() + assert body.closes == 1 + + body.gate.set() + await reader + assert b"".join(received) == b"".join(OPEN_DRIP) + assert body.closes == 1 + + +def test_handed_on_tail_is_bounded_by_the_request_deadline(): + clock = FakeClock() + body = _DripStream(clock, 0.6, OPEN_DRIP) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + assert response.status_code == 503 + + # The three peeked chunks are replayed; the fourth would be waited for + # past the deadline, so the read ends with a timeout there. + chunks = response.iter_bytes() + assert [next(chunks) for _ in range(3)] == OPEN_DRIP[:3] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + next(chunks) + assert body.closes == 1 + + +async def test_async_handed_on_tail_is_bounded_by_the_request_deadline(): + clock = FakeClock() + body = _AsyncDripStream(clock, 0.6, OPEN_DRIP) + inner = AsyncStreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + async def sleep(delay): + clock.sleep(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=clock.monotonic + ).handle_async_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + assert response.status_code == 503 + + received = [] + with pytest.raises(httpx.ReadTimeout, match="reading the response body"): + async for chunk in response.aiter_bytes(): + received.append(chunk) + assert received == OPEN_DRIP[:3] + assert body.closes == 1 + + +BUSY_DRIP = [b'{"code":503,"message":"bu', b"sy", b'"}', b"\n"] + + +def test_a_complete_busy_body_in_the_cut_off_chunk_is_retried_without_reading_the_tail(): + clock = FakeClock() + body = _DripStream(clock, 0.6, BUSY_DRIP) + inner = StreamTransport( + [ + lambda request: httpx.Response(503, stream=body, headers=JSON), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # The third chunk's wait began past the budget but completed the JSON: a + # plain busy 503 read in full, retried like any other with the tail left. + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert body.closes == 1 + + +def test_a_complete_busy_body_in_the_cut_off_chunk_with_no_budget_left_is_returned_whole(): + clock = FakeClock() + body = _DripStream(clock, 0.6, BUSY_DRIP) + inner = StreamTransport( + [lambda request: httpx.Response(503, stream=body, headers=JSON)] + ) + + response = RetryableTransport( + inner, retries=3, sleep=clock.sleep, monotonic=clock.monotonic + ).handle_request( + httpx.Request( + "POST", + "https://api.test/sandboxes/sbx-1/pause", + content=b"{}", + extensions=BUDGET_1S, + ) + ) + + assert response.status_code == 503 + assert response.json() == {"code": 503, "message": "busy"} + assert len(inner.requests) == 1 + assert body.closes == 1 + + +def test_rebuilt_refusal_drops_transfer_headers_and_native_extensions(): + body = _ChunkedStream([b'{"error_code":"sandbox_pause_refused"}']) + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=body, + headers={**JSON, "Transfer-Encoding": "chunked", "X-Keep": "1"}, + extensions={ + "http_version": b"HTTP/2", + "get_trailers": lambda: {"bound": "to the closed response"}, + }, + ), + ] + ) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert "transfer-encoding" not in response.headers + assert response.headers["x-keep"] == "1" + assert response.extensions == {"http_version": b"HTTP/2"} + assert response.request.url.path == "/sandboxes/sbx-1/pause" + + +def test_time_spent_reading_the_503_body_counts_towards_the_delay(): + clock = FakeClock() + inner = StreamTransport( + [ + lambda request: httpx.Response( + 503, + stream=_SlowStream(clock, 0.02, [b'{"code":503}']), + headers={"Content-Length": "12", "Content-Type": "application/json"}, + ), + lambda request: httpx.Response(200, json={}), + ] + ) + + response = RetryableTransport( + inner, + retries=3, + sleep=clock.sleep, + monotonic=clock.monotonic, + random_=lambda: 0.0, + ).handle_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + # Backoff 0.05 s at attempt 0, minus the 0.02 s the read took. + assert response.status_code == 200 + assert len(inner.requests) == 2 + assert clock.sleeps == [pytest.approx(0.03)] + + +def test_service_unavailable_with_another_error_code_is_retried(): + inner = JsonTransport([(503, CAPACITY), (200, {})]) + + response = RetryableTransport( + inner, retries=3, sleep=lambda _: None, monotonic=lambda: 0.0 + ).handle_request(httpx.Request("POST", "https://api.test/sandboxes", content=b"p")) + + assert response.status_code == 200 + assert len(inner.requests) == 2 + + +async def test_async_service_unavailable_with_a_non_retryable_error_code_is_returned_after_one_request(): + inner = JsonAsyncTransport([(503, REFUSED)]) + sleeps = [] + + async def sleep(delay): + sleeps.append(delay) + + response = await AsyncRetryableTransport( + inner, retries=3, sleep=sleep, monotonic=lambda: 0.0 + ).handle_async_request( + httpx.Request("POST", "https://api.test/sandboxes/sbx-1/pause", content=b"{}") + ) + + assert response.status_code == 503 + assert response.json()["error_code"] == "sandbox_pause_refused" + assert len(inner.requests) == 1 + assert sleeps == [] + + def test_service_unavailable_is_retried_for_non_replayable_operation(): inner = FakeTransport([503, 200], retry_after=None)