feat(sdk): retry control-plane requests on Should-Retry, falling back to 429/502/503/504

.changeset/should-retry-header.md

@@ -0,0 +1,9 @@
+---
+'e2b': minor
+'@e2b/python-sdk': minor
+---
+
+Retry control-plane API errors marked `Should-Retry: true`, or 429/502/503/504 responses without the header (e.g. from a load balancer); `Should-Retry: false` is never retried.
+
+- Removes method and replayable-operation retry allowlists
+- Post-send network errors are no longer retried; connection failures still are

packages/js-sdk/src/connectionConfig.ts

@@ -100,21 +100,18 @@ export interface ConnectionOpts {
    */
   requestTimeoutMs?: number
   /**
-   * Number of control-plane API retries after a 429, 502 or 503 response or
-   * a failure to establish the connection (refused, DNS, unreachable host).
-   * Any other network error (dropped connection, opaque browser/Workers
-   * `TypeError`) is retried too. A 502 and those network errors are not
-   * retried for operations that create a resource (e.g. sandbox creation) or
-   * append to one (secret update), as the server may already have processed
-   * them.
-   * A 429 is retried only with a valid, non-negative integer delta-seconds
-   * `Retry-After` header (HTTP-date and malformed values are not retried).
-   * 502 and 503 honor such a `Retry-After` when present; otherwise they and
-   * connection failures use exponential backoff with jitter starting at
-   * 100 ms (capped at 10 s).
+   * Number of control-plane API retries for responses marked `Should-Retry: true`,
+   * or 429/502/503/504 responses without that header. `Should-Retry: false` is
+   * never retried. Waits for a valid `Retry-After` delta-seconds value, otherwise
+   * exponential backoff with jitter from 100 ms to 10 s. Also retries failures
+   * to establish the connection; network errors after the request is sent are
+   * not retried.
+   * A retry is skipped when its wait would exhaust the request timeout.
    * Retry waits use a 60-second total limit when request timeouts are disabled.
+   * Set to `0` to disable retries. Defaults to 3 retries.
    *
    * @default 3
+   * @throws InvalidArgumentError if the value is negative or not an integer
    */
   retries?: number
   /**

packages/js-sdk/src/retry.ts

@@ -1,12 +1,14 @@
 import { InvalidArgumentError } from './errors'
 import { isReadableStreamLike } from './is'
 
+// Retries API responses marked `Should-Retry: true`, or selected statuses
+// when the header is absent.
 const MAX_RETRY_AFTER_SECONDS = 2_147_483
 const MAX_RETRY_WAIT_WITHOUT_TIMEOUT_MS = 60_000
 const BACKOFF_BASE_MS = 100
 const BACKOFF_MAX_MS = 10_000
 const BACKOFF_JITTER_MIN = 0.5
-const RETRYABLE_STATUSES = new Set([429, 502, 503])
+const RETRYABLE_STATUSES = new Set([429, 502, 503, 504])
 // Error codes raised while establishing the connection, before any request
 // bytes are written: Node/undici (`ECONNREFUSED`, `ENOTFOUND`, ...), Bun
 // (`ConnectionRefused`). Post-write failures such as `ECONNRESET` are
@@ -29,25 +31,6 @@ const DENO_CONNECTION_ERROR = /client error \(Connect\)/
 // Chrome `Failed to fetch`, Firefox `NetworkError when attempting to fetch resource.`, Safari `Load failed`
 const OPAQUE_NETWORK_ERROR =
   /^(Failed to fetch|Load failed|NetworkError when attempting to fetch resource\.?)$/
-// GET/PUT/PATCH/DELETE are idempotent by HTTP semantics. A POST is retried
-// after a 502 or a network error that may have occurred once the request was
-// written only if it is listed here: a replay is a no-op or fails with a
-// 404/409 the SDK already tolerates. POSTs that mint a resource without a
-// client-supplied idempotency key (sandbox/fork/snapshot/API-key/volume/secret/
-// webhook creation) or append to one (secret update) must stay off the list —
-// a replay could create a duplicate — and so does any new POST until it is
-// reviewed. Connection-establishment failures
-// are retried for every operation regardless: the request never left.
-const REPLAYABLE_METHODS = new Set(['GET', 'PUT', 'PATCH', 'DELETE'])
-const REPLAYABLE_OPERATIONS: [method: string, path: RegExp][] = [
-  ['POST', /^\/sandboxes\/[^/]+\/(pause|resume|timeout|refreshes)$/],
-  ['POST', /^\/(v2\/)?sandboxes\/[^/]+\/connect$/],
-  ['POST', /^\/v3\/templates$/],
-  ['POST', /^\/v2\/templates\/[^/]+\/builds\/[^/]+$/],
-  ['POST', /^\/templates\/tags$/],
-  ['POST', /^\/nodes\/[^/]+$/],
-  ['POST', /^\/admin\/teams\/[^/]+\/(sandboxes\/kill|builds\/cancel)$/],
-]
 
 export function resolveRetries(retries: number): number {
   if (!Number.isInteger(retries) || retries < 0) {
@@ -104,31 +87,12 @@ export function isNetworkError(error: unknown): boolean {
 }
 
 /**
- * Whether `request` may be sent again after a 502 or a network error that may
- * have occurred once the request was written (see `REPLAYABLE_OPERATIONS`).
+ * Whether a `fetch` rejection is a connection-establishment failure. Aborts
+ * and failures after the request may have been sent are not retried.
  */
-export function isReplayable(request: Request): boolean {
-  if (REPLAYABLE_METHODS.has(request.method)) return true
-  const { pathname } = new URL(request.url)
-  return REPLAYABLE_OPERATIONS.some(
-    ([method, path]) => request.method === method && path.test(pathname)
-  )
-}
-
-/**
- * Whether a `fetch` rejection may be retried. Connection-establishment
- * failures are always retried: the request never left. For a replayable
- * request any other network error is retried as well — a connection dropped
- * mid-request, or the opaque `TypeError` browsers and Cloudflare Workers raise
- * for every network failure. Aborts are never retried.
- */
-export function isRetryableFetchError(
-  error: unknown,
-  replayable: boolean
-): boolean {
+export function isRetryableFetchError(error: unknown): boolean {
   if (error instanceof DOMException) return false
-  if (isConnectionError(error)) return true
-  return replayable && error instanceof Error
+  return isConnectionError(error)
 }
 
 type RetryDependencies = {
@@ -161,33 +125,32 @@ function backoffMs(attempt: number, random: () => number): number {
 }
 
 /**
- * Delay before the next attempt, or `undefined` when the response is not
- * retried: `Retry-After` when the server sends a usable one, otherwise
- * exponential backoff with jitter for 502/503. A 429 without `Retry-After`
- * is not retried, nor is a 502 for a non-replayable request: a gateway may
- * answer 502 after the backend already processed it.
+ * Delay before the next attempt, or `undefined` unless `Should-Retry` is true
+ * or the header is absent for a retryable status. A present header is
+ * authoritative, including explicit `false`.
  */
 function retryDelayMs(
   response: Response,
   attempt: number,
-  replayable: boolean,
   random: () => number
 ): number | undefined {
-  if (!RETRYABLE_STATUSES.has(response.status)) return undefined
-  if (response.status === 502 && !replayable) return undefined
+  const shouldRetry = response.headers.get('Should-Retry')
+  const retryable =
+    shouldRetry === null
+      ? RETRYABLE_STATUSES.has(response.status)
+      : shouldRetry.trim().toLowerCase() === 'true'
+  if (!retryable) return undefined
 
   const retryAfter = parseRetryAfter(response.headers.get('Retry-After'))
   if (retryAfter !== undefined) return retryAfter * 1000
-  if (response.status === 429) return undefined
 
   return backoffMs(attempt, random)
 }
 
 /**
- * Retry requests after a 429 carrying `Retry-After`, a 503 or — for replayable
- * requests ({@link isReplayable}) — a 502 (using `Retry-After` when present,
- * exponential backoff otherwise), or a network failure (exponential backoff;
- * see {@link isRetryableFetchError}).
+ * Retries API responses marked `Should-Retry: true`, or 429/502/503/504
+ * responses without the header, after `Retry-After` or exponential backoff.
+ * Also retries connection-establishment failures.
  */
 export function withRetry(
   fetchImpl: typeof fetch,
@@ -216,7 +179,6 @@ export function withRetry(
         : new Request(input, init)
     const deadline =
       monotonic() + (requestTimeoutMs || MAX_RETRY_WAIT_WITHOUT_TIMEOUT_MS)
-    const replayable = isReplayable(request)
 
     for (let attempt = 0; ; attempt++) {
       let response: Response
@@ -228,7 +190,7 @@ export function withRetry(
         if (
           attempt === retries ||
           request.signal.aborted ||
-          !isRetryableFetchError(error, replayable)
+          !isRetryableFetchError(error)
         ) {
           throw error
         }
@@ -241,7 +203,7 @@ export function withRetry(
       }
       if (attempt === retries) return response
 
-      const delayMs = retryDelayMs(response, attempt, replayable, random)
+      const delayMs = retryDelayMs(response, attempt, random)
       if (delayMs === undefined || monotonic() + delayMs >= deadline) {
         return response
       }

packages/js-sdk/tests/client.test.ts

@@ -176,7 +176,7 @@ test('client retries rate-limited control-plane requests', async () => {
       if (attempts === 1) {
         return new HttpResponse(null, {
           status: 429,
-          headers: { 'Retry-After': '0' },
+          headers: { 'Should-Retry': 'true', 'Retry-After': '0' },
         })
       }
       return HttpResponse.json([])
@@ -200,7 +200,7 @@ test('client replays serialized control-plane JSON after a rate limit', async ()
       if (bodies.length === 1) {
         return new HttpResponse(null, {
           status: 429,
-          headers: { 'Retry-After': '0' },
+          headers: { 'Should-Retry': 'true', 'Retry-After': '0' },
         })
       }
       return HttpResponse.json(sandboxResponse)
@@ -223,7 +223,10 @@ test('client retries sandbox creation after a 503', async () => {
       if (attempts === 1) {
         return HttpResponse.json(
           { code: 503, message: 'not enough capacity' },
-          { status: 503, headers: { 'Retry-After': '0' } }
+          {
+            status: 503,
+            headers: { 'Should-Retry': 'true', 'Retry-After': '0' },
+          }
         )
       }
       return HttpResponse.json(sandboxResponse)