diff --git a/packages/envd/internal/api/init.go b/packages/envd/internal/api/init.go index 85eb09bb8d..14f2d8fe0e 100644 --- a/packages/envd/internal/api/init.go +++ b/packages/envd/internal/api/init.go @@ -33,6 +33,7 @@ var pinMMDSWarnLimit = ratelimit.New(10 * time.Second) var ( ErrAccessTokenMismatch = errors.New("access token validation failed") ErrAccessTokenResetNotAuthorized = errors.New("access token reset not authorized") + ErrConcurrentNFSInit = errors.New("NFS mount already in progress") ) const ( @@ -334,6 +335,12 @@ func writeInitError(w http.ResponseWriter, logger zerolog.Logger, err error) { switch { case errors.Is(err, ErrAccessTokenMismatch), errors.Is(err, ErrAccessTokenResetNotAuthorized): w.WriteHeader(http.StatusUnauthorized) + case errors.Is(err, ErrConcurrentNFSInit): + // Transient: a previous /init is still mounting NFS (setupNFS uses + // context.WithoutCancel so it outlives a request cancelled by EnvdInitRequestTimeout). + // The orchestrator retries on 503. + logger.Warn().Err(err).Msg("NFS mount already in progress, retrying") + w.WriteHeader(http.StatusServiceUnavailable) case errors.Is(err, host.ErrCAInstallInProgress): // Not a failure, a concurrent install still holds the CA lock. logger.Warn().Err(err).Msg("CA initialization still in progress, retrying") @@ -371,7 +378,7 @@ func (a *API) setupNFS(ctx context.Context, logger zerolog.Logger, lifecycleID * if !a.isMountingNFS.CompareAndSwap(false, true) { logger.Debug().Msg("NFS volumes already mounting") - return e + return ErrConcurrentNFSInit } defer a.isMountingNFS.Store(false) diff --git a/packages/envd/pkg/version.go b/packages/envd/pkg/version.go index 5e5d110c55..bc489ba2bd 100644 --- a/packages/envd/pkg/version.go +++ b/packages/envd/pkg/version.go @@ -1,3 +1,3 @@ package pkg -const Version = "0.6.13" // x-release-please-version +const Version = "0.6.14" // x-release-please-version diff --git a/packages/orchestrator/pkg/sandbox/envd.go b/packages/orchestrator/pkg/sandbox/envd.go index 18e6b5324f..e1febc17a5 100644 --- a/packages/orchestrator/pkg/sandbox/envd.go +++ b/packages/orchestrator/pkg/sandbox/envd.go @@ -118,7 +118,14 @@ func (s *Sandbox) doRequestWithInfiniteRetries( cancel() if err == nil { - return response, requestCount, nil + // Treat 503 as a transient failure and retry: envd returns 503 when a + // concurrent /init is still in progress (e.g. an NFS mount outlived the + // previous per-request timeout). Discard the body and loop. + if response.StatusCode == http.StatusServiceUnavailable { + response.Body.Close() + } else { + return response, requestCount, nil + } } select {