From ac6364798caadfe3e8232c98277d2122a147e38d Mon Sep 17 00:00:00 2001 From: AdaAibaby Date: Thu, 30 Jul 2026 17:22:01 +0800 Subject: [PATCH 1/3] fix(envd): return error when concurrent /init races on NFS mount setupNFS guards against concurrent callers with an atomic CAS, but on contention it returned nil (the named return zero value). The second caller then continued as if NFS was mounted, so the sandbox started with missing volume mounts and no indication of failure. Return an explicit error so the caller receives a 500 and can retry, rather than silently succeeding with no mount. --- packages/envd/internal/api/init.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 85eb09bb8d..360c6e64fb 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -33,6 +33,7 @@ var pinMMDSWarnLimit = ratelimit.New(10 * time.Second) var ( ErrAccessTokenMismatch = errors.New("access token validation failed") ErrAccessTokenResetNotAuthorized = errors.New("access token reset not authorized") + ErrConcurrentNFSInit = errors.New("NFS mount already in progress") ) const ( @@ -371,7 +372,7 @@ func (a *API) setupNFS(ctx context.Context, logger zerolog.Logger, lifecycleID * if !a.isMountingNFS.CompareAndSwap(false, true) { logger.Debug().Msg("NFS volumes already mounting") - return e + return ErrConcurrentNFSInit } defer a.isMountingNFS.Store(false) From eaba58c2a367253718341092dc32c7c9bce499c4 Mon Sep 17 00:00:00 2001 From: AdaAibaby Date: Tue, 4 Aug 2026 17:29:08 +0800 Subject: [PATCH 2/3] chore(envd): bump version to 0.6.14 --- packages/envd/pkg/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 5e5d110c55..bc489ba2bd 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.6.13" // x-release-please-version +const Version = "0.6.14" // x-release-please-version From 851478c7749225719409b7703fe786d99d52b0de Mon Sep 17 00:00:00 2001 From: AdaAibaby Date: Tue, 4 Aug 2026 17:34:58 +0800 Subject: [PATCH 3/3] fix(envd): return 503 on concurrent NFS init so orchestrator retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ErrConcurrentNFSInit was mapped to HTTP 400 in writeInitError, but doRequestWithInfiniteRetries only retries on network errors, not on successful HTTP responses. So when a slow NFS mount (> EnvdInitRequestTimeout) kept isMountingNFS set while the orchestrator retried the per-request timeout, the second /init got a 400, initEnvd declared the sandbox start a failure, and the sandbox was aborted. Fix both sides: - envd: map ErrConcurrentNFSInit to 503 (consistent with ErrCAInstallInProgress, which already used 503 for the same "transient, retry" semantics) - orchestrator: treat a 503 response from envd as a transient error in doRequestWithInfiniteRetries — close the body and loop, exactly as for a network-level error --- packages/envd/internal/api/init.go | 6 ++++++ packages/orchestrator/pkg/sandbox/envd.go | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 360c6e64fb..14f2d8fe0e 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -335,6 +335,12 @@ func writeInitError(w http.ResponseWriter, logger zerolog.Logger, err error) { switch { case errors.Is(err, ErrAccessTokenMismatch), errors.Is(err, ErrAccessTokenResetNotAuthorized): w.WriteHeader(http.StatusUnauthorized) + case errors.Is(err, ErrConcurrentNFSInit): + // Transient: a previous /init is still mounting NFS (setupNFS uses + // context.WithoutCancel so it outlives a request cancelled by EnvdInitRequestTimeout). + // The orchestrator retries on 503. + logger.Warn().Err(err).Msg("NFS mount already in progress, retrying") + w.WriteHeader(http.StatusServiceUnavailable) case errors.Is(err, host.ErrCAInstallInProgress): // Not a failure, a concurrent install still holds the CA lock. logger.Warn().Err(err).Msg("CA initialization still in progress, retrying") diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index 18e6b5324f..e1febc17a5 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -118,7 +118,14 @@ func (s *Sandbox) doRequestWithInfiniteRetries( cancel() if err == nil { - return response, requestCount, nil + // Treat 503 as a transient failure and retry: envd returns 503 when a + // concurrent /init is still in progress (e.g. an NFS mount outlived the + // previous per-request timeout). Discard the body and loop. + if response.StatusCode == http.StatusServiceUnavailable { + response.Body.Close() + } else { + return response, requestCount, nil + } } select {