diff --git a/binding/nvml/library_device.go b/binding/nvml/library_device.go index 2c29698e1..68ce17973 100644 --- a/binding/nvml/library_device.go +++ b/binding/nvml/library_device.go @@ -232,9 +232,10 @@ func (l Device) GetNumGpuCores() (uint32, Return) { return numCores, ret } -// GetMemoryBusWidth retrieves the device's memory bus width in bits. It distinguishes HBM -// (wide bus: data-center stacks are >=1024-bit) from GDDR (<=384-bit), which is what decides -// whether enabling ECC costs user-visible memory. +// GetMemoryBusWidth retrieves the device's memory bus width in bits. It is the detector's first +// discriminator for the ECC capacity rule: data-center HBM stacks are >=1024-bit while GDDR +// parts (GDDR7 included) top out at 512-bit. Whether enabling ECC mode also costs user-visible +// memory depends on the generation, which the detector reads separately. func (l Device) GetMemoryBusWidth() (uint32, Return) { if l.so.Lookup("nvmlDeviceGetMemoryBusWidth") != nil { return 0, ERROR_FUNCTION_NOT_FOUND diff --git a/pkg/devicemanager/detector/nvidia/device.go b/pkg/devicemanager/detector/nvidia/device.go index bfa4438d9..48a28c4c4 100644 --- a/pkg/devicemanager/detector/nvidia/device.go +++ b/pkg/devicemanager/detector/nvidia/device.go @@ -21,9 +21,14 @@ const Manufacturer = nodefeature.ManufacturerNVIDIA // hbmMemoryBusWidthBits is the memory bus width (bits) at or above which a GPU is HBM rather // than GDDR. Data-center HBM stacks are >=1024-bit (e.g. A30 3072, A100/H100 5120) while GDDR -// data-center parts top out at 384-bit, so the threshold separates them with a wide margin. +// parts top out at 512-bit (GDDR7 included), so the threshold separates them with a wide margin. const hbmMemoryBusWidthBits = 1024 +// gddr7ComputeCapabilityMajor is the first generation whose GDDR memory keeps ECC inside the +// DRAM die (GDDR7 on RTX Blackwell) instead of trading ~1/16 of the framebuffer for ECC-mode +// parity, which the restore in DetectAccelerator compensates on earlier GDDR generations. +const gddr7ComputeCapabilityMajor = 12 + var _PciVendor string func init() { @@ -132,6 +137,10 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList, continue } + // Read before the memory block: the ECC capacity rule below gates on the generation, and + // the group fields after the block reuse the same reading. + ccMajor, ccMinor, ccRet := dev.GetCudaComputeCapability() + var ( memory uint64 memoryUnhealthy bool @@ -146,18 +155,27 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList, continue } } + // NVML reports the total with the current ECC mode already applied. memory = device.ConvertBytesToMiB(memInfo.Total) - // ECC parity bits are carved out of user-visible memory only on GDDR - // GPUs; HBM keeps ECC in a hardware-reserved region and already reports - // full capacity. The memory bus width is the NVML-native discriminator — - // data-center HBM stacks are >=1024-bit (H100 5120, A100 5120, A30 3072) - // while GDDR parts are <=384-bit — so restore the ~1/16 ECC loss (to the - // physical/marketing size) only on a narrow (GDDR) bus with ECC enabled. - // When the bus width is unreadable (older driver) no restore is applied. + // Enabling ECC mode costs user-visible capacity on GDDR parts that keep ECC parity + // outside the framebuffer: since Turing, professional GDDR cards give ~1/16 of memory + // to parity when the mode is on, so restore the nominal size (x16/15) there. Two + // boundaries keep the rule honest. HBM never restores: its ECC lives in + // hardware-reserved regions and NVML always reports full capacity, and data-center HBM + // stacks are >=1024-bit (A30 3072, A100/H100 5120) while GDDR parts top out at 512-bit. + // And GDDR7 never restores: its ECC is built into the DRAM die (NVIDIA RTX Blackwell + // GPU Architecture whitepaper v1.1), so an ECC-enabled RTX PRO 6000 Blackwell Server + // Edition reports its 96 GB nominal as 97887 MiB total -- measured, driver 580.173.02 + // -- which the restore would inflate to 104412 MiB. RTX Blackwell is compute capability + // 12.0, so the restore applies only below that major. A reading the detector cannot + // classify -- bus width, ECC mode, or compute capability unreadable -- keeps the + // driver-reported total rather than a capacity the driver did not report. if bw, ret := dev.GetMemoryBusWidth(); ret.IsSuccess() && bw > 0 && bw < hbmMemoryBusWidthBits { if cur, _, ret := dev.GetEccMode(); ret.IsSuccess() && cur == nvml.FEATURE_ENABLED { - memory = memory * 16 / 15 + if ccRet.IsSuccess() && ccMajor < gddr7ComputeCapabilityMajor { + memory = memory * 16 / 15 + } } } @@ -171,8 +189,6 @@ func (in *nvidia) DetectAccelerator(noPciCheck bool) (_ device.DevicesGroupList, } } - ccMajor, ccMinor, _ := dev.GetCudaComputeCapability() - grpIndex := slices.IndexFunc(grpList, func(grp device.DevicesGroup) bool { return grp.Name == name && grp.Memory == memory }) diff --git a/pkg/devicemanager/detector/nvidia/device_test.go b/pkg/devicemanager/detector/nvidia/device_test.go new file mode 100644 index 000000000..a4d6c4667 --- /dev/null +++ b/pkg/devicemanager/detector/nvidia/device_test.go @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 GPUStack, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package nvidia + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "gpustack.ai/gpustack/pkg/device" + "gpustack.ai/gpustack/pkg/nodefeature" +) + +// TestDetectAcceleratorMemory drives the real DetectAccelerator and Node-label construction +// against a simulated NVML driver in a subprocess, one fixture library per case. The capacity +// cases encode the ECC memory rule: NVML reports the total with the current ECC mode applied, +// and the nominal size is restored only on pre-GDDR7 GDDR parts with ECC enabled -- measured on +// an ECC-enabled RTX PRO 6000 Blackwell Server Edition (driver 580.173.02): 97887 MiB total, +// 512-bit bus, compute capability 12.0, no ECC carve. +func TestDetectAcceleratorMemory(t *testing.T) { + // Each subprocess loads its own NVML fixture without affecting other tests. + if os.Getenv("GPUSTACK_TEST_NVML_MEMORY") == "true" { + groups, err := New(device.DetectorOptions{}).DetectAccelerator(true) + require.NoError(t, err) + require.Len(t, groups, 1) + want, err := strconv.ParseUint(os.Getenv("GPUSTACK_TEST_NVML_WANT_MIB"), 10, 64) + require.NoError(t, err) + assert.Equal(t, want, groups[0].Memory) + require.Len(t, groups[0].Accelerators, 1) + labels := nodefeature.ConstructAcceleratableNodeLabels(groups) + key := nodefeature.AcceleratableFeatureLabelPrefix + Manufacturer + "-" + groups[0].ID + ".memory" + assert.Equal(t, os.Getenv("GPUSTACK_TEST_NVML_DISPLAY_MEMORY"), labels[key]) + return + } + + dir := t.TempDir() + flags := make([]string, 0, 7) + if runtime.GOOS == "darwin" { + flags = append(flags, "-dynamiclib") + } else { + flags = append(flags, "-shared", "-fPIC") + } + flags = append(flags, "-DNVML_NO_UNVERSIONED_FUNC_DEFS=1", "-I../../../../binding/nvml", + "testdata/memory_nvml.c", "-o", filepath.Join(dir, "libnvidia-ml.so.1")) + output, err := exec.Command("cc", flags...).CombinedOutput() + require.NoError(t, err, "%s", output) + + cases := []struct { + name string + memory uint64 // fixture's MemoryInfo total, in MiB + want uint64 // expected DevicesGroup.Memory after the ECC rule, in MiB + display string // expected Node memory label + busWidth uint32 + busErr bool + ecc bool + eccErr bool + ccMajor int32 + ccMinor int32 + ccErr bool + v1 bool + }{ + // The measured RTX PRO 6000 readings. GDDR7 keeps ECC inside the DRAM die, so the + // driver-reported total is already nominal and must not be restored (baseline bug: 102Gi). + { + name: "RTX PRO 6000 GDDR7 ECC", memory: 97887, want: 97887, display: "96Gi", + busWidth: 512, ecc: true, ccMajor: 12, ccMinor: 0, + }, + // L40S-class GDDR6: enabling ECC mode takes ~1/16 of the framebuffer, which the restore + // gives back (broad-deletion regression: 45Gi). + { + name: "L40S GDDR6 ECC", memory: 46080, want: 49152, display: "48Gi", + busWidth: 384, ecc: true, ccMajor: 8, ccMinor: 9, + }, + // Another pre-GDDR7 generation on a narrower bus: the restore must not be tuned to 512-bit. + { + name: "Turing T4 GDDR6 ECC", memory: 15360, want: 16384, display: "16Gi", + busWidth: 256, ecc: true, ccMajor: 7, ccMinor: 5, + }, + // A narrow GDDR7 part: the generation, not the bus width, ends the restore. + { + name: "narrow GDDR7", memory: 8192, want: 8192, display: "8Gi", + busWidth: 128, ecc: true, ccMajor: 12, ccMinor: 0, + }, + // ECC mode off: nothing was taken, nothing to restore. + { + name: "GDDR ECC disabled", memory: 49152, want: 49152, display: "48Gi", + busWidth: 384, ccMajor: 8, ccMinor: 9, + }, + // HBM: ECC lives in hardware-reserved regions; NVML always reports full capacity. + { + name: "HBM ECC", memory: 81559, want: 81559, display: "80Gi", + busWidth: 5120, ecc: true, ccMajor: 9, ccMinor: 0, + }, + // Classifications the detector cannot make keep the driver-reported total. + { + name: "zero memory bus", memory: 46080, want: 46080, display: "45Gi", + busWidth: 0, ecc: true, ccMajor: 8, ccMinor: 9, + }, + { + name: "memory bus unreadable", memory: 46080, want: 46080, display: "45Gi", + busErr: true, ecc: true, ccMajor: 8, ccMinor: 9, + }, + { + name: "ECC mode unreadable", memory: 46080, want: 46080, display: "45Gi", + busWidth: 384, eccErr: true, ccMajor: 8, ccMinor: 9, + }, + { + name: "compute capability unreadable", memory: 46080, want: 46080, display: "45Gi", + busWidth: 384, ecc: true, ccErr: true, + }, + // The V1 memory fallback rides the same rule. + { + name: "memory V1 fallback restores GDDR6", memory: 46080, want: 49152, display: "48Gi", + busWidth: 384, ecc: true, ccMajor: 8, ccMinor: 9, v1: true, + }, + { + name: "memory V1 fallback keeps GDDR7", memory: 97887, want: 97887, display: "96Gi", + busWidth: 512, ecc: true, ccMajor: 12, ccMinor: 0, v1: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestDetectAcceleratorMemory$", "-test.count=1") + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GPUSTACK_TEST_NVML_MEMORY=true", + "GPUSTACK_TEST_NVML_TOTAL_MIB="+strconv.FormatUint(tc.memory, 10), + "GPUSTACK_TEST_NVML_WANT_MIB="+strconv.FormatUint(tc.want, 10), + "GPUSTACK_TEST_NVML_DISPLAY_MEMORY="+tc.display, + "GPUSTACK_TEST_NVML_BUS_WIDTH="+strconv.FormatUint(uint64(tc.busWidth), 10), + "GPUSTACK_TEST_NVML_BUS_WIDTH_ERROR="+strconv.FormatBool(tc.busErr), + "GPUSTACK_TEST_NVML_ECC="+strconv.FormatBool(tc.ecc), + "GPUSTACK_TEST_NVML_ECC_ERROR="+strconv.FormatBool(tc.eccErr), + "GPUSTACK_TEST_NVML_CC_MAJOR="+strconv.FormatInt(int64(tc.ccMajor), 10), + "GPUSTACK_TEST_NVML_CC_MINOR="+strconv.FormatInt(int64(tc.ccMinor), 10), + "GPUSTACK_TEST_NVML_CC_ERROR="+strconv.FormatBool(tc.ccErr), + "GPUSTACK_TEST_NVML_V1="+strconv.FormatBool(tc.v1), + "LD_LIBRARY_PATH="+dir, + "DYLD_LIBRARY_PATH="+dir, + ) + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + }) + } +} diff --git a/pkg/devicemanager/detector/nvidia/testdata/memory_nvml.c b/pkg/devicemanager/detector/nvidia/testdata/memory_nvml.c new file mode 100644 index 000000000..40fd55769 --- /dev/null +++ b/pkg/devicemanager/detector/nvidia/testdata/memory_nvml.c @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2026 GPUStack, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "nvml.h" +#include +#include +#include + +nvmlReturn_t nvmlInit_v2(void) { return NVML_SUCCESS; } + +const char *nvmlErrorString(nvmlReturn_t result) { return "NVML fixture"; } + +nvmlReturn_t nvmlDeviceGetCount_v2(unsigned int *count) { + *count = 1; + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetHandleByIndex_v2(unsigned int index, nvmlDevice_t *device) { + device->handle = (struct nvmlDevice_st *)1; + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetUUID(nvmlDevice_t device, char *uuid, unsigned int length) { + snprintf(uuid, length, "GPU-memory-fixture"); + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetPciInfo_v2(nvmlDevice_t device, nvmlPciInfo_t *pci) { + snprintf(pci->busId, sizeof(pci->busId), "0000:01:00.0"); + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetName(nvmlDevice_t device, char *name, unsigned int length) { + snprintf(name, length, "NVIDIA memory fixture"); + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetMemoryInfo_v2(nvmlDevice_t device, nvmlMemory_v2_t *memory) { + if (strcmp(getenv("GPUSTACK_TEST_NVML_V1"), "true") == 0) + return NVML_ERROR_NOT_SUPPORTED; + memory->total = strtoull(getenv("GPUSTACK_TEST_NVML_TOTAL_MIB"), NULL, 10) << 20; + memory->free = memory->total; + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetMemoryInfo(nvmlDevice_t device, nvmlMemory_t *memory) { + memory->total = strtoull(getenv("GPUSTACK_TEST_NVML_TOTAL_MIB"), NULL, 10) << 20; + memory->free = memory->total; + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetMemoryBusWidth(nvmlDevice_t device, unsigned int *width) { + if (strcmp(getenv("GPUSTACK_TEST_NVML_BUS_WIDTH_ERROR"), "true") == 0) + return NVML_ERROR_NOT_SUPPORTED; + *width = strtoul(getenv("GPUSTACK_TEST_NVML_BUS_WIDTH"), NULL, 10); + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetEccMode(nvmlDevice_t device, nvmlEnableState_t *current, + nvmlEnableState_t *pending) { + if (strcmp(getenv("GPUSTACK_TEST_NVML_ECC_ERROR"), "true") == 0) + return NVML_ERROR_NOT_SUPPORTED; + *current = strcmp(getenv("GPUSTACK_TEST_NVML_ECC"), "true") == 0 + ? NVML_FEATURE_ENABLED : NVML_FEATURE_DISABLED; + *pending = *current; + return NVML_SUCCESS; +} + +nvmlReturn_t nvmlDeviceGetCudaComputeCapability(nvmlDevice_t device, int *major, int *minor) { + if (strcmp(getenv("GPUSTACK_TEST_NVML_CC_ERROR"), "true") == 0) + return NVML_ERROR_NOT_SUPPORTED; + *major = atoi(getenv("GPUSTACK_TEST_NVML_CC_MAJOR")); + *minor = atoi(getenv("GPUSTACK_TEST_NVML_CC_MINOR")); + return NVML_SUCCESS; +}