nfsproxy: make NFS file-handle cache limit configurable via NFS_PROXY_CACHE_LIMIT

packages/api/internal/handlers/sandbox_kill.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
+	"go.uber.org/zap"
 
 	"github.com/e2b-dev/infra/packages/api/internal/db"
 	"github.com/e2b-dev/infra/packages/api/internal/orchestrator"
@@ -69,12 +70,26 @@ func (a *APIStore) DeleteSandboxesSandboxID(
 	case err == nil:
 		killedOrRemoved = true
 	case errors.Is(err, orchestrator.ErrSandboxNotFound):
-		logger.L().Debug(ctx, "Running sandbox not found", logger.WithSandboxID(sandboxID))
+		logger.L().Info(ctx, "Running sandbox not found, checking for snapshot",
+			logger.WithSandboxID(sandboxID),
+			logger.WithTeamID(teamID.String()),
+		)
 	case errors.Is(err, orchestrator.ErrSandboxOperationFailed):
+		logger.L().Error(ctx, "Failed to kill sandbox on orchestrator node",
+			logger.WithSandboxID(sandboxID),
+			logger.WithTeamID(teamID.String()),
+			zap.Error(err),
+		)
+		telemetry.ReportError(ctx, "error killing sandbox on node", err)
 		a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error killing sandbox: %s", err))
 
 		return
 	default:
+		logger.L().Error(ctx, "Unexpected error removing sandbox",
+			logger.WithSandboxID(sandboxID),
+			logger.WithTeamID(teamID.String()),
+			zap.Error(err),
+		)
 		telemetry.ReportError(ctx, "error killing sandbox", err)
 		a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error killing sandbox: %s", err))
 
@@ -87,7 +102,12 @@ func (a *APIStore) DeleteSandboxesSandboxID(
 	case errors.Is(deleteSnapshotErr, db.ErrSnapshotNotFound):
 		// no snapshot found, nothing to do
 	case deleteSnapshotErr != nil:
-		telemetry.ReportError(ctx, "error deleting sandbox", deleteSnapshotErr)
+		logger.L().Error(ctx, "Failed to delete sandbox snapshot from DB",
+			logger.WithSandboxID(sandboxID),
+			logger.WithTeamID(teamID.String()),
+			zap.Error(deleteSnapshotErr),
+		)
+		telemetry.ReportError(ctx, "error deleting sandbox snapshot", deleteSnapshotErr)
 		a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error deleting sandbox: %s", deleteSnapshotErr))
 
 		return
@@ -98,7 +118,10 @@ func (a *APIStore) DeleteSandboxesSandboxID(
 	if killedOrRemoved {
 		c.Status(http.StatusNoContent)
 	} else {
-		logger.L().Debug(ctx, "Sandbox not found for deletion", logger.WithSandboxID(sandboxID))
+		logger.L().Info(ctx, "Sandbox not found: not running and no snapshot",
+			logger.WithSandboxID(sandboxID),
+			logger.WithTeamID(teamID.String()),
+		)
 		a.sendAPIStoreError(c, http.StatusNotFound, utils.SandboxNotFoundMsg(sandboxID))
 	}
 }

packages/orchestrator/cmd/inspect-nbd/main.go

@@ -0,0 +1,102 @@
+//go:build linux
+
+// inspect-nbd lists the NBD devices that are currently connected on this host.
+//
+// Background: the nbd kernel module is loaded with nbds_max=4096, creating
+// 4096 /dev/nbdX entries in the OS even when only a handful are in use.
+// `lsblk | grep nbd` produces 4096 lines of noise. This tool reads the kernel's
+// /sys/block/nbdX/pid sentinel — which only exists for connected devices — and
+// prints a concise table of active slots.
+//
+// This is an operator debugging tool, not a service. It is not part of the
+// orchestrator Nomad job and is not uploaded to GCS. To use it on a sandbox
+// node, compile it locally and copy the binary:
+//
+//	GOOS=linux GOARCH=amd64 go build -o /tmp/inspect-nbd ./cmd/inspect-nbd
+//	scp /tmp/inspect-nbd <node>:/tmp/inspect-nbd
+//
+// Usage:
+//
+//	inspect-nbd [-json]
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/nbd"
+)
+
+func main() {
+	jsonOut := flag.Bool("json", false, "output as JSON")
+	flag.Parse()
+
+	maxData, err := os.ReadFile("/sys/module/nbd/parameters/nbds_max")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: nbd module not loaded or /sys unavailable: %v\n", err)
+		os.Exit(1)
+	}
+	nbdsMax, _ := strconv.ParseUint(strings.TrimSpace(string(maxData)), 10, 64)
+
+	devices, err := nbd.ConnectedDevices()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: failed to list connected devices: %v\n", err)
+		os.Exit(1)
+	}
+
+	type deviceInfo struct {
+		Slot   uint32 `json:"slot"`
+		Path   string `json:"path"`
+		SizeMB uint64 `json:"size_mb"`
+		PID    string `json:"pid"`
+	}
+
+	infos := make([]deviceInfo, 0, len(devices))
+	for _, slot := range devices {
+		sizeRaw, _ := os.ReadFile(fmt.Sprintf("/sys/block/nbd%d/size", slot))
+		sectors, _ := strconv.ParseUint(strings.TrimSpace(string(sizeRaw)), 10, 64)
+
+		pidRaw, _ := os.ReadFile(fmt.Sprintf("/sys/block/nbd%d/pid", slot))
+
+		infos = append(infos, deviceInfo{
+			Slot:   slot,
+			Path:   nbd.GetDevicePath(slot),
+			SizeMB: (sectors * 512) / (1024 * 1024),
+			PID:    strings.TrimSpace(string(pidRaw)),
+		})
+	}
+
+	if *jsonOut {
+		type output struct {
+			NBDsMax   uint64       `json:"nbds_max"`
+			Connected int          `json:"connected"`
+			Devices   []deviceInfo `json:"devices"`
+		}
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(output{NBDsMax: nbdsMax, Connected: len(infos), Devices: infos})
+
+		return
+	}
+
+	fmt.Printf("NBD devices: %d connected / %d configured\n", len(infos), nbdsMax)
+
+	if len(infos) == 0 {
+		fmt.Println("No connected NBD devices.")
+
+		return
+	}
+
+	fmt.Println()
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "SLOT\tDEVICE\tSIZE (MB)\tPID")
+	for _, info := range infos {
+		fmt.Fprintf(w, "%d\t%s\t%d\t%s\n", info.Slot, info.Path, info.SizeMB, info.PID)
+	}
+	_ = w.Flush()
+}

packages/orchestrator/pkg/cfg/model.go

@@ -95,6 +95,11 @@ type Config struct {
 	NFSProxyRecordHandleCalls   bool              `env:"NFS_PROXY_RECORD_HANDLE_CALLS" envDefault:"false"`
 	NFSProxyRecordStatCalls     bool              `env:"NFS_PROXY_RECORD_STAT_CALLS"   envDefault:"false"`
 	NFSProxyLogLevel            nfs.LogLevel      `env:"NFS_PROXY_LOG_LEVEL"           envDefault:"info"`
+	// NFSProxyCacheLimit controls the LRU size for NFS file-handle mappings.
+	// Too small a value causes ESTALE errors when a sandbox creates more files than
+	// the cache can hold simultaneously (e.g. large npm/pip installs).
+	// Each entry is ~150 bytes; 16384 entries ≈ 2.4 MB per orchestrator node.
+	NFSProxyCacheLimit          int               `env:"NFS_PROXY_CACHE_LIMIT"         envDefault:"16384"`
 	ProxyPort                   uint16            `env:"PROXY_PORT"                    envDefault:"5007"`
 	RedisClusterURL             string            `env:"REDIS_CLUSTER_URL"`
 	RedisTLSCABase64            string            `env:"REDIS_TLS_CA_BASE64"`

packages/orchestrator/pkg/factories/run.go

@@ -1040,6 +1040,7 @@ func startNFSProxy(
 		RecordHandleCalls: config.NFSProxyRecordHandleCalls,
 		RecordStatCalls:   config.NFSProxyRecordStatCalls,
 		NFSLogLevel:       config.NFSProxyLogLevel,
+		CacheLimit:        config.NFSProxyCacheLimit,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create nfs proxy: %w", err)

packages/orchestrator/pkg/nfsproxy/cfg/model.go

@@ -9,4 +9,8 @@ type Config struct {
 	RecordStatCalls   bool
 	RecordHandleCalls bool
 	NFSLogLevel       nfs.LogLevel
+	// CacheLimit is the LRU capacity for NFS file-handle mappings (uuid → path).
+	// When the cache is full the oldest entry is evicted; subsequent RPCs using
+	// that handle return ESTALE. Set via NFS_PROXY_CACHE_LIMIT env var.
+	CacheLimit int
 }

packages/orchestrator/pkg/nfsproxy/proxy.go

@@ -22,8 +22,6 @@ import (
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox"
 )
 
-const cacheLimit = 1024
-
 var setLogLevelOnce sync.Once
 
 type Proxy struct {
@@ -47,7 +45,7 @@ func NewProxy(ctx context.Context, builder *chrooted.Builder, sandboxes *sandbox
 	}
 
 	// wrap the handler in middleware
-	handler = helpers.NewCachingHandler(handler, cacheLimit)
+	handler = helpers.NewCachingHandler(handler, config.CacheLimit)
 
 	if config.Tracing {
 		handler = tracing.WrapWithTracing(handler, config)

packages/orchestrator/pkg/sandbox/nbd/pool.go

@@ -79,13 +79,38 @@ type DevicePool struct {
 	done     chan struct{}
 	doneOnce sync.Once
 
+	maxDevices uint
+
 	// We use the bitset to speedup the free device lookup.
 	usedSlots *bitset.BitSet
 	mu        sync.Mutex
 
 	slots chan DeviceSlot
 }
 
+// PoolStatus summarises the current state of the device pool.
+type PoolStatus struct {
+	// Max is the total number of NBD device slots available on this host (nbds_max).
+	Max uint
+	// Used is the number of slots currently held by the pool.
+	Used uint
+	// PreWarmed is the number of slots pre-warmed and immediately available.
+	PreWarmed int
+}
+
+// Status returns a snapshot of the pool's current usage without scanning sysfs.
+func (d *DevicePool) Status() PoolStatus {
+	d.mu.Lock()
+	used := d.usedSlots.Count()
+	d.mu.Unlock()
+
+	return PoolStatus{
+		Max:       d.maxDevices,
+		Used:      used,
+		PreWarmed: len(d.slots),
+	}
+}
+
 func NewDevicePool(maxSlotsReady int) (*DevicePool, error) {
 	if maxSlotsReady <= 0 {
 		return nil, fmt.Errorf("maxSlotsReady must be > 0, got %d", maxSlotsReady)
@@ -101,9 +126,10 @@ func NewDevicePool(maxSlotsReady int) (*DevicePool, error) {
 	}
 
 	pool := &DevicePool{
-		done:      make(chan struct{}),
-		usedSlots: bitset.New(maxDevices),
-		slots:     make(chan DeviceSlot, int(math.Min(float64(maxSlotsReady), float64(maxDevices)))),
+		done:       make(chan struct{}),
+		maxDevices: maxDevices,
+		usedSlots:  bitset.New(maxDevices),
+		slots:      make(chan DeviceSlot, int(math.Min(float64(maxSlotsReady), float64(maxDevices)))),
 	}
 
 	return pool, nil