fix(devicemanager): stop restoring ECC capacity on GDDR7 GPUs

binding/nvml/library_device.go

@@ -232,9 +232,10 @@ func (l Device) GetNumGpuCores() (uint32, Return) {
 	return numCores, ret
 }
 
-// GetMemoryBusWidth retrieves the device's memory bus width in bits. It distinguishes HBM
-// (wide bus: data-center stacks are >=1024-bit) from GDDR (<=384-bit), which is what decides
-// whether enabling ECC costs user-visible memory.
+// GetMemoryBusWidth retrieves the device's memory bus width in bits. It is the detector's first
+// discriminator for the ECC capacity rule: data-center HBM stacks are >=1024-bit while GDDR
+// parts (GDDR7 included) top out at 512-bit. Whether enabling ECC mode also costs user-visible
+// memory depends on the generation, which the detector reads separately.
 func (l Device) GetMemoryBusWidth() (uint32, Return) {
 	if l.so.Lookup("nvmlDeviceGetMemoryBusWidth") != nil {
 		return 0, ERROR_FUNCTION_NOT_FOUND

pkg/devicemanager/detector/nvidia/device.go

@@ -21,9 +21,14 @@ const Manufacturer = nodefeature.ManufacturerNVIDIA
 
 // hbmMemoryBusWidthBits is the memory bus width (bits) at or above which a GPU is HBM rather
 // than GDDR. Data-center HBM stacks are >=1024-bit (e.g. A30 3072, A100/H100 5120) while GDDR
-// data-center parts top out at 384-bit, so the threshold separates them with a wide margin.
+// parts top out at 512-bit (GDDR7 included), so the threshold separates them with a wide margin.
 const hbmMemoryBusWidthBits = 1024
 
+// gddr7ComputeCapabilityMajor is the first generation whose GDDR memory keeps ECC inside the
+// DRAM die (GDDR7 on RTX Blackwell) instead of trading ~1/16 of the framebuffer for ECC-mode
+// parity, which the restore in DetectAccelerator compensates on earlier GDDR generations.
+const gddr7ComputeCapabilityMajor = 12
+
 var _PciVendor string
 
 func init() {
@@ -132,6 +137,10 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList,
 			continue
 		}
 
+		// Read before the memory block: the ECC capacity rule below gates on the generation, and
+		// the group fields after the block reuse the same reading.
+		ccMajor, ccMinor, ccRet := dev.GetCudaComputeCapability()
+
 		var (
 			memory          uint64
 			memoryUnhealthy bool
@@ -146,18 +155,27 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList,
 					continue
 				}
 			}
+			// NVML reports the total with the current ECC mode already applied.
 			memory = device.ConvertBytesToMiB(memInfo.Total)
 
-			// ECC parity bits are carved out of user-visible memory only on GDDR
-			// GPUs; HBM keeps ECC in a hardware-reserved region and already reports
-			// full capacity. The memory bus width is the NVML-native discriminator —
-			// data-center HBM stacks are >=1024-bit (H100 5120, A100 5120, A30 3072)
-			// while GDDR parts are <=384-bit — so restore the ~1/16 ECC loss (to the
-			// physical/marketing size) only on a narrow (GDDR) bus with ECC enabled.
-			// When the bus width is unreadable (older driver) no restore is applied.
+			// Enabling ECC mode costs user-visible capacity on GDDR parts that keep ECC parity
+			// outside the framebuffer: since Turing, professional GDDR cards give ~1/16 of memory
+			// to parity when the mode is on, so restore the nominal size (x16/15) there. Two
+			// boundaries keep the rule honest. HBM never restores: its ECC lives in
+			// hardware-reserved regions and NVML always reports full capacity, and data-center HBM
+			// stacks are >=1024-bit (A30 3072, A100/H100 5120) while GDDR parts top out at 512-bit.
+			// And GDDR7 never restores: its ECC is built into the DRAM die (NVIDIA RTX Blackwell
+			// GPU Architecture whitepaper v1.1), so an ECC-enabled RTX PRO 6000 Blackwell Server
+			// Edition reports its 96 GB nominal as 97887 MiB total -- measured, driver 580.173.02
+			// -- which the restore would inflate to 104412 MiB. RTX Blackwell is compute capability
+			// 12.0, so the restore applies only below that major. A reading the detector cannot
+			// classify -- bus width, ECC mode, or compute capability unreadable -- keeps the
+			// driver-reported total rather than a capacity the driver did not report.
 			if bw, ret := dev.GetMemoryBusWidth(); ret.IsSuccess() && bw > 0 && bw < hbmMemoryBusWidthBits {
 				if cur, _, ret := dev.GetEccMode(); ret.IsSuccess() && cur == nvml.FEATURE_ENABLED {
-					memory = memory * 16 / 15
+					if ccRet.IsSuccess() && ccMajor < gddr7ComputeCapabilityMajor {
+						memory = memory * 16 / 15
+					}
 				}
 			}
 
@@ -171,8 +189,6 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList,
 			}
 		}
 
-		ccMajor, ccMinor, _ := dev.GetCudaComputeCapability()
-
 		grpIndex := slices.IndexFunc(grpList, func(grp device.DevicesGroup) bool {
 			return grp.Name == name && grp.Memory == memory
 		})

pkg/devicemanager/detector/nvidia/device_test.go

@@ -0,0 +1,152 @@
+// SPDX-FileCopyrightText: 2026 GPUStack, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package nvidia
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"gpustack.ai/gpustack/pkg/device"
+	"gpustack.ai/gpustack/pkg/nodefeature"
+)
+
+// TestDetectAcceleratorMemory drives the real DetectAccelerator and Node-label construction
+// against a simulated NVML driver in a subprocess, one fixture library per case. The capacity
+// cases encode the ECC memory rule: NVML reports the total with the current ECC mode applied,
+// and the nominal size is restored only on pre-GDDR7 GDDR parts with ECC enabled -- measured on
+// an ECC-enabled RTX PRO 6000 Blackwell Server Edition (driver 580.173.02): 97887 MiB total,
+// 512-bit bus, compute capability 12.0, no ECC carve.
+func TestDetectAcceleratorMemory(t *testing.T) {
+	// Each subprocess loads its own NVML fixture without affecting other tests.
+	if os.Getenv("GPUSTACK_TEST_NVML_MEMORY") == "true" {
+		groups, err := New(device.DetectorOptions{}).DetectAccelerator(true)
+		require.NoError(t, err)
+		require.Len(t, groups, 1)
+		want, err := strconv.ParseUint(os.Getenv("GPUSTACK_TEST_NVML_WANT_MIB"), 10, 64)
+		require.NoError(t, err)
+		assert.Equal(t, want, groups[0].Memory)
+		require.Len(t, groups[0].Accelerators, 1)
+		labels := nodefeature.ConstructAcceleratableNodeLabels(groups)
+		key := nodefeature.AcceleratableFeatureLabelPrefix + Manufacturer + "-" + groups[0].ID + ".memory"
+		assert.Equal(t, os.Getenv("GPUSTACK_TEST_NVML_DISPLAY_MEMORY"), labels[key])
+		return
+	}
+
+	dir := t.TempDir()
+	flags := make([]string, 0, 7)
+	if runtime.GOOS == "darwin" {
+		flags = append(flags, "-dynamiclib")
+	} else {
+		flags = append(flags, "-shared", "-fPIC")
+	}
+	flags = append(flags, "-DNVML_NO_UNVERSIONED_FUNC_DEFS=1", "-I../../../../binding/nvml",
+		"testdata/memory_nvml.c", "-o", filepath.Join(dir, "libnvidia-ml.so.1"))
+	output, err := exec.Command("cc", flags...).CombinedOutput()
+	require.NoError(t, err, "%s", output)
+
+	cases := []struct {
+		name     string
+		memory   uint64 // fixture's MemoryInfo total, in MiB
+		want     uint64 // expected DevicesGroup.Memory after the ECC rule, in MiB
+		display  string // expected Node memory label
+		busWidth uint32
+		busErr   bool
+		ecc      bool
+		eccErr   bool
+		ccMajor  int32
+		ccMinor  int32
+		ccErr    bool
+		v1       bool
+	}{
+		// The measured RTX PRO 6000 readings. GDDR7 keeps ECC inside the DRAM die, so the
+		// driver-reported total is already nominal and must not be restored (baseline bug: 102Gi).
+		{
+			name: "RTX PRO 6000 GDDR7 ECC", memory: 97887, want: 97887, display: "96Gi",
+			busWidth: 512, ecc: true, ccMajor: 12, ccMinor: 0,
+		},
+		// L40S-class GDDR6: enabling ECC mode takes ~1/16 of the framebuffer, which the restore
+		// gives back (broad-deletion regression: 45Gi).
+		{
+			name: "L40S GDDR6 ECC", memory: 46080, want: 49152, display: "48Gi",
+			busWidth: 384, ecc: true, ccMajor: 8, ccMinor: 9,
+		},
+		// Another pre-GDDR7 generation on a narrower bus: the restore must not be tuned to 512-bit.
+		{
+			name: "Turing T4 GDDR6 ECC", memory: 15360, want: 16384, display: "16Gi",
+			busWidth: 256, ecc: true, ccMajor: 7, ccMinor: 5,
+		},
+		// A narrow GDDR7 part: the generation, not the bus width, ends the restore.
+		{
+			name: "narrow GDDR7", memory: 8192, want: 8192, display: "8Gi",
+			busWidth: 128, ecc: true, ccMajor: 12, ccMinor: 0,
+		},
+		// ECC mode off: nothing was taken, nothing to restore.
+		{
+			name: "GDDR ECC disabled", memory: 49152, want: 49152, display: "48Gi",
+			busWidth: 384, ccMajor: 8, ccMinor: 9,
+		},
+		// HBM: ECC lives in hardware-reserved regions; NVML always reports full capacity.
+		{
+			name: "HBM ECC", memory: 81559, want: 81559, display: "80Gi",
+			busWidth: 5120, ecc: true, ccMajor: 9, ccMinor: 0,
+		},
+		// Classifications the detector cannot make keep the driver-reported total.
+		{
+			name: "zero memory bus", memory: 46080, want: 46080, display: "45Gi",
+			busWidth: 0, ecc: true, ccMajor: 8, ccMinor: 9,
+		},
+		{
+			name: "memory bus unreadable", memory: 46080, want: 46080, display: "45Gi",
+			busErr: true, ecc: true, ccMajor: 8, ccMinor: 9,
+		},
+		{
+			name: "ECC mode unreadable", memory: 46080, want: 46080, display: "45Gi",
+			busWidth: 384, eccErr: true, ccMajor: 8, ccMinor: 9,
+		},
+		{
+			name: "compute capability unreadable", memory: 46080, want: 46080, display: "45Gi",
+			busWidth: 384, ecc: true, ccErr: true,
+		},
+		// The V1 memory fallback rides the same rule.
+		{
+			name: "memory V1 fallback restores GDDR6", memory: 46080, want: 49152, display: "48Gi",
+			busWidth: 384, ecc: true, ccMajor: 8, ccMinor: 9, v1: true,
+		},
+		{
+			name: "memory V1 fallback keeps GDDR7", memory: 97887, want: 97887, display: "96Gi",
+			busWidth: 512, ecc: true, ccMajor: 12, ccMinor: 0, v1: true,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			cmd := exec.Command(os.Args[0], "-test.run=^TestDetectAcceleratorMemory$", "-test.count=1")
+			cmd.Dir = dir
+			cmd.Env = append(os.Environ(),
+				"GPUSTACK_TEST_NVML_MEMORY=true",
+				"GPUSTACK_TEST_NVML_TOTAL_MIB="+strconv.FormatUint(tc.memory, 10),
+				"GPUSTACK_TEST_NVML_WANT_MIB="+strconv.FormatUint(tc.want, 10),
+				"GPUSTACK_TEST_NVML_DISPLAY_MEMORY="+tc.display,
+				"GPUSTACK_TEST_NVML_BUS_WIDTH="+strconv.FormatUint(uint64(tc.busWidth), 10),
+				"GPUSTACK_TEST_NVML_BUS_WIDTH_ERROR="+strconv.FormatBool(tc.busErr),
+				"GPUSTACK_TEST_NVML_ECC="+strconv.FormatBool(tc.ecc),
+				"GPUSTACK_TEST_NVML_ECC_ERROR="+strconv.FormatBool(tc.eccErr),
+				"GPUSTACK_TEST_NVML_CC_MAJOR="+strconv.FormatInt(int64(tc.ccMajor), 10),
+				"GPUSTACK_TEST_NVML_CC_MINOR="+strconv.FormatInt(int64(tc.ccMinor), 10),
+				"GPUSTACK_TEST_NVML_CC_ERROR="+strconv.FormatBool(tc.ccErr),
+				"GPUSTACK_TEST_NVML_V1="+strconv.FormatBool(tc.v1),
+				"LD_LIBRARY_PATH="+dir,
+				"DYLD_LIBRARY_PATH="+dir,
+			)
+			output, err := cmd.CombinedOutput()
+			require.NoError(t, err, "%s", output)
+		})
+	}
+}

pkg/devicemanager/detector/nvidia/testdata/memory_nvml.c

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2026 GPUStack, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "nvml.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+nvmlReturn_t nvmlInit_v2(void) { return NVML_SUCCESS; }
+
+const char *nvmlErrorString(nvmlReturn_t result) { return "NVML fixture"; }
+
+nvmlReturn_t nvmlDeviceGetCount_v2(unsigned int *count) {
+    *count = 1;
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetHandleByIndex_v2(unsigned int index, nvmlDevice_t *device) {
+    device->handle = (struct nvmlDevice_st *)1;
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetUUID(nvmlDevice_t device, char *uuid, unsigned int length) {
+    snprintf(uuid, length, "GPU-memory-fixture");
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetPciInfo_v2(nvmlDevice_t device, nvmlPciInfo_t *pci) {
+    snprintf(pci->busId, sizeof(pci->busId), "0000:01:00.0");
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetName(nvmlDevice_t device, char *name, unsigned int length) {
+    snprintf(name, length, "NVIDIA memory fixture");
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetMemoryInfo_v2(nvmlDevice_t device, nvmlMemory_v2_t *memory) {
+    if (strcmp(getenv("GPUSTACK_TEST_NVML_V1"), "true") == 0)
+        return NVML_ERROR_NOT_SUPPORTED;
+    memory->total = strtoull(getenv("GPUSTACK_TEST_NVML_TOTAL_MIB"), NULL, 10) << 20;
+    memory->free = memory->total;
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetMemoryInfo(nvmlDevice_t device, nvmlMemory_t *memory) {
+    memory->total = strtoull(getenv("GPUSTACK_TEST_NVML_TOTAL_MIB"), NULL, 10) << 20;
+    memory->free = memory->total;
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetMemoryBusWidth(nvmlDevice_t device, unsigned int *width) {
+    if (strcmp(getenv("GPUSTACK_TEST_NVML_BUS_WIDTH_ERROR"), "true") == 0)
+        return NVML_ERROR_NOT_SUPPORTED;
+    *width = strtoul(getenv("GPUSTACK_TEST_NVML_BUS_WIDTH"), NULL, 10);
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetEccMode(nvmlDevice_t device, nvmlEnableState_t *current,
+                                nvmlEnableState_t *pending) {
+    if (strcmp(getenv("GPUSTACK_TEST_NVML_ECC_ERROR"), "true") == 0)
+        return NVML_ERROR_NOT_SUPPORTED;
+    *current = strcmp(getenv("GPUSTACK_TEST_NVML_ECC"), "true") == 0
+        ? NVML_FEATURE_ENABLED : NVML_FEATURE_DISABLED;
+    *pending = *current;
+    return NVML_SUCCESS;
+}
+
+nvmlReturn_t nvmlDeviceGetCudaComputeCapability(nvmlDevice_t device, int *major, int *minor) {
+    if (strcmp(getenv("GPUSTACK_TEST_NVML_CC_ERROR"), "true") == 0)
+        return NVML_ERROR_NOT_SUPPORTED;
+    *major = atoi(getenv("GPUSTACK_TEST_NVML_CC_MAJOR"));
+    *minor = atoi(getenv("GPUSTACK_TEST_NVML_CC_MINOR"));
+    return NVML_SUCCESS;
+}