frost: fix oversized-SMEM query crashing all frost GEMM on CUDA<13.4 cuda-python #615
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -119,20 +119,22 @@ def shared_memory_per_block_optin(device: int) -> int: | |
| return int(_ck(*drv.cuDeviceGetAttribute(drv.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, handle))) | ||
|
|
||
|
|
||
| @functools.lru_cache(maxsize=None) | ||
| def oversized_shared_memory_per_block(device: int) -> int: | ||
| """Per-CTA SMEM ceiling in the *oversized* carveout (327 KiB vs the 227 KiB | ||
| opt-in limit on SM 10.7), which the part gives by shrinking L1 to 8 kB — free for | ||
| a TMA-fed GEMM. 0 when the driver has no such mode.""" | ||
| drv = _driver() | ||
| # CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK arrived in CUDA 13.4. | ||
| # A driver older than that has no such mode -> 0 by design (not an error), and we | ||
| # do not touch the enum member (which an older cuda-python's CUdevice_attribute | ||
| # does not carry -- passing a bare ordinal would raise, since the binding reads | ||
| # attrib.value). From 13.4 the attribute is real: query it and let a genuine | ||
| # failure raise rather than masking it as 0. | ||
| if int(_ck(*drv.cuDriverGetVersion())) < 13040: | ||
| return 0 | ||
| handle = _device_handle(device) | ||
| return int(_ck(*drv.cuDeviceGetAttribute(drv.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK, handle))) | ||
|
Contributor
There was a problem hiding this comment. 🩺 Stability & Availability | 🟠 Major | ⚡ Quick win Preserve the Line 137 sends every non-zero Proposed error handling- return int(_ck(*drv.cuDeviceGetAttribute(drv.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK, handle)))
+ err, value = drv.cuDeviceGetAttribute(
+ drv.CUdevice_attribute.CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK, handle
+ )
+ if int(err) == int(drv.CUresult.CUDA_ERROR_INVALID_VALUE):
+ return 0
+ return int(_ck(err, value))🤖 Prompt for AI AgentsSource: MCP tools 🎯 Functional Correctness | 🟠 Major | 🏗️ Heavy lift 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
rg -n -C 4 'cuda-python|cuda\.bindings|MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK' pyproject.toml python test 2>/dev/null || true
python - <<'PY'
from importlib import metadata
name = "CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK"
try:
import cuda.bindings.driver as drv
except Exception as exc:
print(f"Unable to inspect cuda.bindings.driver: {exc}")
else:
try:
version = metadata.version("cuda-python")
except metadata.PackageNotFoundError:
version = "distribution version unavailable"
print(f"cuda-python: {version}")
print(f"enum present: {hasattr(drv.CUdevice_attribute, name)}")
PYRepository: NVIDIA/cudnn-frontend Length of output: 50378 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- pyproject dependency declarations ---'
rg -n -C 5 'cuda-python|cuda\.bindings' pyproject.toml setup.cfg setup.py 2>/dev/null || true
printf '%s\n' '--- affected helper and nearby callers ---'
cat -n python/cudnn/frost/device.py | sed -n '1,180p'
rg -n -C 5 'oversized_shared_memory_per_block|MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK|_device_handle|_driver' python test --glob '*.py' --glob '!**/csrc/**' | head -n 400
printf '%s\n' '--- tracked dependency metadata files ---'
git ls-files | rg '(^|/)(pyproject\.toml|requirements[^/]*|environment[^/]*|setup\.cfg|setup\.py|Pipfile|poetry\.lock|uv\.lock|conda)' | head -n 200Repository: NVIDIA/cudnn-frontend Length of output: 39984 🏁 Script executed: #!/bin/bash
set -euo pipefail
python3 - <<'PY'
from pathlib import Path
for path in [Path("pyproject.toml"), Path("python/cudnn/frost/device.py")]:
print(f"--- {path} ---")
text = path.read_text()
lines = text.splitlines()
needles = ("cuda-python", "MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK",
"def oversized_shared_memory_per_block", "def _driver",
"def _device_handle")
for i, line in enumerate(lines, 1):
if any(n in line for n in needles):
lo, hi = max(1, i - 8), min(len(lines), i + 12)
for j in range(lo, hi + 1):
print(f"{j}: {lines[j-1]}")
print()
PY
printf '%s\n' '--- public develop manifest at the referenced location ---'
curl -L --fail --silent --show-error \
https://raw.githubusercontent.com/NVIDIA/cudnn-frontend/develop/pyproject.toml \
| rg -n -C 4 'cuda-python|cuda\.bindings' || trueRepository: NVIDIA/cudnn-frontend Length of output: 7677 🏁 Script executed: #!/bin/bash
set -euo pipefail
python3 - <<'PY'
from pathlib import Path
p = Path("python/cudnn/frost/device.py")
lines = p.read_text().splitlines()
for start, end in [(1, 155)]:
for i in range(start, min(end, len(lines)) + 1):
print(f"{i}: {lines[i-1]}")
PYRepository: NVIDIA/cudnn-frontend Length of output: 6830 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- dependency declaration ---'
rg -n -C 8 'cuda-python|cuda\.bindings' pyproject.toml 2>/dev/null || true
printf '%s\n' '--- helper implementation ---'
cat -n python/cudnn/frost/device.py | sed -n '1,155p'
printf '%s\n' '--- helper usages ---'
rg -n -C 8 'oversized_shared_memory_per_block|MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK' python test --glob '*.py' | head -n 300Repository: NVIDIA/cudnn-frontend Length of output: 12760 🌐 Web query:
💡 Result: The attribute CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK is not a standard part of the official public CUDA Driver API header files (e.g., cuda.h). Instead, it appears to be a custom or internal constant used in certain third-party projects, such as the Triton compiler, where it is manually defined with a value of 150 [1]. Because this attribute is not an official member of the CUDA Driver API, there is no official version in which it was introduced. Projects that utilize this name typically define it manually as a fallback or experimental mechanism to query device capabilities, often in conjunction with CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN [1]. You should not expect to find this attribute in standard NVIDIA CUDA documentation or official libraries. Citations: 🌐 Web query:
💡 Result: There is no attribute named CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK in the cuda-python library or the underlying CUDA Driver API [1][2][3]. It is likely that this name is a misunderstanding of how CUDA handles shared memory, specifically the distinction between standard shared memory limits and "opt-in" shared memory [1][4][3]. In CUDA, you can request shared memory beyond the standard per-block limit (up to a device-specific maximum) using dynamic shared memory allocation [5][6]. The relevant attributes in cuda-python and the CUDA API are: 1. CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK: The standard maximum amount of shared memory available to a thread block [1][2]. 2. CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN: The absolute maximum amount of shared memory a block can use if you explicitly opt-in to using larger amounts [1][3][7]. 3. CU_DEVICE_ATTRIBUTE_RESERVED_SHARED_MEMORY_PER_BLOCK: The amount of shared memory per block that is reserved by the CUDA driver [1][3][5]. If you are trying to determine the amount of shared memory available for a kernel, you should typically query these attributes and compare them against your kernel's requirements [5][7][8]. Failing to stay within these bounds will generally result in a synchronous kernel launch failure [9]. Citations:
🌐 Web query:
💡 Result: The enumerator CU_DEVICE_ATTRIBUTE_MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK is included in the CUDA 13.4 Driver API [1]. It is defined in the CUDA header files (cuda.h) to represent the maximum oversized shared memory available per thread block [1]. This attribute is utilized in contexts where applications need to query the maximum allowed shared memory for configurations that exceed standard limitations, often functioning as an alternative or supplementary check to the standard CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK [2][3]. Technical implementations occasionally use this attribute within cuDeviceGetAttribute calls to determine hardware capabilities for dynamic or specialized shared memory allocations [3]. Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- repository references and numeric fallbacks ---'
rg -n -C 5 'MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK|13040|oversized_shared_memory' . --glob '!*.ipynb' --glob '!build/**' --glob '!dist/**' | head -n 300Repository: NVIDIA/cudnn-frontend Length of output: 4086 🏁 Script executed: #!/bin/bash
set -euo pipefail
python3 - <<'PY'
import json
import urllib.request
urls = [
"https://api.github.com/repos/NVIDIA/cuda-python/contents",
"https://pypi.org/pypi/cuda-python/json",
]
for url in urls:
print(f"--- {url} ---")
try:
req = urllib.request.Request(url, headers={"User-Agent": "cudnn-frontend-review"})
with urllib.request.urlopen(req, timeout=20) as response:
data = json.load(response)
if "pypi.org" in url:
info = data["info"]
print("latest:", info.get("version"))
print("home_page:", info.get("home_page"))
print("project_urls:", info.get("project_urls"))
print("releases:", sorted(data.get("releases", {}))[-10:])
else:
print([(item.get("name"), item.get("type"), item.get("download_url")) for item in data])
except Exception as exc:
print(type(exc).__name__, exc)
PY
printf '%s\n' '--- upstream source references ---'
for url in \
https://raw.githubusercontent.com/NVIDIA/cuda-python/main/cuda/bindings/driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/master/cuda/bindings/driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/main/cuda/bindings/_driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/master/cuda/bindings/_driver.py
do
printf '%s\n' "--- $url ---"
curl -L --fail --silent --show-error "$url" | rg -n -C 3 'MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK|CUdevice_attribute' | head -n 80 || true
doneRepository: NVIDIA/cudnn-frontend Length of output: 1156 🏁 Script executed: #!/bin/bash
set -euo pipefail
python3 - <<'PY'
import json
import urllib.request
for url in [
"https://api.github.com/repos/NVIDIA/cuda-python/contents",
"https://pypi.org/pypi/cuda-python/json",
]:
print(f"--- {url} ---")
try:
req = urllib.request.Request(url, headers={"User-Agent": "cudnn-frontend-review"})
with urllib.request.urlopen(req, timeout=20) as response:
data = json.load(response)
if "pypi.org" in url:
print("latest:", data["info"].get("version"))
print("project_urls:", data["info"].get("project_urls"))
print("recent releases:", sorted(data.get("releases", {}))[-10:])
else:
print([(x.get("name"), x.get("type"), x.get("download_url")) for x in data])
except Exception as exc:
print(type(exc).__name__, exc)
PY
for url in \
https://raw.githubusercontent.com/NVIDIA/cuda-python/main/cuda/bindings/driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/master/cuda/bindings/driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/main/cuda/bindings/_driver.py \
https://raw.githubusercontent.com/NVIDIA/cuda-python/master/cuda/bindings/_driver.py
do
printf '%s\n' "--- $url ---"
curl -L --fail --silent --show-error "$url" \
| rg -n -C 3 'MAX_OVERSIZED_SHARED_MEMORY_PER_BLOCK|CUdevice_attribute' \
| head -n 80 || true
doneRepository: NVIDIA/cudnn-frontend Length of output: 1121 Do not gate enum availability only on the driver version.
🤖 Prompt for AI AgentsSource: MCP tools |
||
|
|
||
|
|
||
| @functools.lru_cache(maxsize=None) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win
Validate the device before the version short-circuit.
Line 134 dereferences
drvbefore_device_handle(device)can validate it. When_driver()returnsNone, this raisesAttributeErrorinstead of the existingRuntimeError. On an older driver, an invalid device ordinal also returns0without validation. Move_device_handle(device)before the version check.Proposed ordering
📝 Committable suggestion
🤖 Prompt for AI Agents