diff --git a/packages/api/internal/handlers/sandbox_kill.go b/packages/api/internal/handlers/sandbox_kill.go index b30ac6b5a9..031f97ca11 100644 --- a/packages/api/internal/handlers/sandbox_kill.go +++ b/packages/api/internal/handlers/sandbox_kill.go @@ -8,6 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" + "go.uber.org/zap" "github.com/e2b-dev/infra/packages/api/internal/db" "github.com/e2b-dev/infra/packages/api/internal/orchestrator" @@ -69,12 +70,26 @@ func (a *APIStore) DeleteSandboxesSandboxID( case err == nil: killedOrRemoved = true case errors.Is(err, orchestrator.ErrSandboxNotFound): - logger.L().Debug(ctx, "Running sandbox not found", logger.WithSandboxID(sandboxID)) + logger.L().Info(ctx, "Running sandbox not found, checking for snapshot", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID.String()), + ) case errors.Is(err, orchestrator.ErrSandboxOperationFailed): + logger.L().Error(ctx, "Failed to kill sandbox on orchestrator node", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID.String()), + zap.Error(err), + ) + telemetry.ReportError(ctx, "error killing sandbox on node", err) a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error killing sandbox: %s", err)) return default: + logger.L().Error(ctx, "Unexpected error removing sandbox", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID.String()), + zap.Error(err), + ) telemetry.ReportError(ctx, "error killing sandbox", err) a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error killing sandbox: %s", err)) @@ -87,7 +102,12 @@ func (a *APIStore) DeleteSandboxesSandboxID( case errors.Is(deleteSnapshotErr, db.ErrSnapshotNotFound): // no snapshot found, nothing to do case deleteSnapshotErr != nil: - telemetry.ReportError(ctx, "error deleting sandbox", deleteSnapshotErr) + logger.L().Error(ctx, "Failed to delete sandbox snapshot from DB", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID.String()), + zap.Error(deleteSnapshotErr), + ) + telemetry.ReportError(ctx, "error deleting sandbox snapshot", deleteSnapshotErr) a.sendAPIStoreError(c, http.StatusInternalServerError, fmt.Sprintf("Error deleting sandbox: %s", deleteSnapshotErr)) return @@ -98,7 +118,10 @@ func (a *APIStore) DeleteSandboxesSandboxID( if killedOrRemoved { c.Status(http.StatusNoContent) } else { - logger.L().Debug(ctx, "Sandbox not found for deletion", logger.WithSandboxID(sandboxID)) + logger.L().Info(ctx, "Sandbox not found: not running and no snapshot", + logger.WithSandboxID(sandboxID), + logger.WithTeamID(teamID.String()), + ) a.sendAPIStoreError(c, http.StatusNotFound, utils.SandboxNotFoundMsg(sandboxID)) } } diff --git a/packages/orchestrator/cmd/inspect-nbd/main.go b/packages/orchestrator/cmd/inspect-nbd/main.go new file mode 100644 index 0000000000..ca7924e816 --- /dev/null +++ b/packages/orchestrator/cmd/inspect-nbd/main.go @@ -0,0 +1,102 @@ +//go:build linux + +// inspect-nbd lists the NBD devices that are currently connected on this host. +// +// Background: the nbd kernel module is loaded with nbds_max=4096, creating +// 4096 /dev/nbdX entries in the OS even when only a handful are in use. +// `lsblk | grep nbd` produces 4096 lines of noise. This tool reads the kernel's +// /sys/block/nbdX/pid sentinel — which only exists for connected devices — and +// prints a concise table of active slots. +// +// This is an operator debugging tool, not a service. It is not part of the +// orchestrator Nomad job and is not uploaded to GCS. To use it on a sandbox +// node, compile it locally and copy the binary: +// +// GOOS=linux GOARCH=amd64 go build -o /tmp/inspect-nbd ./cmd/inspect-nbd +// scp /tmp/inspect-nbd :/tmp/inspect-nbd +// +// Usage: +// +// inspect-nbd [-json] +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strconv" + "strings" + "text/tabwriter" + + "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/nbd" +) + +func main() { + jsonOut := flag.Bool("json", false, "output as JSON") + flag.Parse() + + maxData, err := os.ReadFile("/sys/module/nbd/parameters/nbds_max") + if err != nil { + fmt.Fprintf(os.Stderr, "error: nbd module not loaded or /sys unavailable: %v\n", err) + os.Exit(1) + } + nbdsMax, _ := strconv.ParseUint(strings.TrimSpace(string(maxData)), 10, 64) + + devices, err := nbd.ConnectedDevices() + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to list connected devices: %v\n", err) + os.Exit(1) + } + + type deviceInfo struct { + Slot uint32 `json:"slot"` + Path string `json:"path"` + SizeMB uint64 `json:"size_mb"` + PID string `json:"pid"` + } + + infos := make([]deviceInfo, 0, len(devices)) + for _, slot := range devices { + sizeRaw, _ := os.ReadFile(fmt.Sprintf("/sys/block/nbd%d/size", slot)) + sectors, _ := strconv.ParseUint(strings.TrimSpace(string(sizeRaw)), 10, 64) + + pidRaw, _ := os.ReadFile(fmt.Sprintf("/sys/block/nbd%d/pid", slot)) + + infos = append(infos, deviceInfo{ + Slot: slot, + Path: nbd.GetDevicePath(slot), + SizeMB: (sectors * 512) / (1024 * 1024), + PID: strings.TrimSpace(string(pidRaw)), + }) + } + + if *jsonOut { + type output struct { + NBDsMax uint64 `json:"nbds_max"` + Connected int `json:"connected"` + Devices []deviceInfo `json:"devices"` + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + _ = enc.Encode(output{NBDsMax: nbdsMax, Connected: len(infos), Devices: infos}) + + return + } + + fmt.Printf("NBD devices: %d connected / %d configured\n", len(infos), nbdsMax) + + if len(infos) == 0 { + fmt.Println("No connected NBD devices.") + + return + } + + fmt.Println() + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "SLOT\tDEVICE\tSIZE (MB)\tPID") + for _, info := range infos { + fmt.Fprintf(w, "%d\t%s\t%d\t%s\n", info.Slot, info.Path, info.SizeMB, info.PID) + } + _ = w.Flush() +} diff --git a/packages/orchestrator/pkg/sandbox/nbd/pool.go b/packages/orchestrator/pkg/sandbox/nbd/pool.go index a5ded0f732..534e6c5590 100644 --- a/packages/orchestrator/pkg/sandbox/nbd/pool.go +++ b/packages/orchestrator/pkg/sandbox/nbd/pool.go @@ -79,6 +79,8 @@ type DevicePool struct { done chan struct{} doneOnce sync.Once + maxDevices uint + // We use the bitset to speedup the free device lookup. usedSlots *bitset.BitSet mu sync.Mutex @@ -86,6 +88,29 @@ type DevicePool struct { slots chan DeviceSlot } +// PoolStatus summarises the current state of the device pool. +type PoolStatus struct { + // Max is the total number of NBD device slots available on this host (nbds_max). + Max uint + // Used is the number of slots currently held by the pool. + Used uint + // PreWarmed is the number of slots pre-warmed and immediately available. + PreWarmed int +} + +// Status returns a snapshot of the pool's current usage without scanning sysfs. +func (d *DevicePool) Status() PoolStatus { + d.mu.Lock() + used := d.usedSlots.Count() + d.mu.Unlock() + + return PoolStatus{ + Max: d.maxDevices, + Used: used, + PreWarmed: len(d.slots), + } +} + func NewDevicePool(maxSlotsReady int) (*DevicePool, error) { if maxSlotsReady <= 0 { return nil, fmt.Errorf("maxSlotsReady must be > 0, got %d", maxSlotsReady) @@ -101,9 +126,10 @@ func NewDevicePool(maxSlotsReady int) (*DevicePool, error) { } pool := &DevicePool{ - done: make(chan struct{}), - usedSlots: bitset.New(maxDevices), - slots: make(chan DeviceSlot, int(math.Min(float64(maxSlotsReady), float64(maxDevices)))), + done: make(chan struct{}), + maxDevices: maxDevices, + usedSlots: bitset.New(maxDevices), + slots: make(chan DeviceSlot, int(math.Min(float64(maxSlotsReady), float64(maxDevices)))), } return pool, nil