gpu_pulse.py

"""Compact always-on-top GPU monitor widget using tkinter + nvidia-smi."""

import ctypes
import json
import subprocess
import tkinter as tk
import urllib.request

BG = "#1e1e1e"
FG = "#cccccc"
FG_DIM = "#888888"
FG_MODEL = "#7eb8da"
BAR_W = 80
BAR_H = 10
VRAM_BAR_W = 60
VRAM_BAR_H = 8
RAM_BAR_W = 100
RAM_BAR_H = 8
REFRESH_MS = 5_000

# Subtle left-border accent colors for baseboard groups
BOARD_COLORS = ["#5b8c5a", "#5a7ebf", "#bf8c5a", "#8c5abf", "#bf5a5a", "#5abfbf"]


def query_gpus():
    """Call nvidia-smi and return list of gpu dicts with VRAM, bus ID, and power limit range."""
    fields_full = (
        "index,name,temperature.gpu,power.draw,power.limit,"
        "utilization.gpu,memory.used,memory.total,pci.bus_id,"
        "power.min_limit,power.max_limit,power.default_power_limit"
    )
    fields_basic = (
        "index,name,temperature.gpu,power.draw,power.limit,"
        "utilization.gpu,memory.used,memory.total,pci.bus_id,"
        "power.min_limit,power.max_limit"
    )
    out = None
    for fields in (fields_full, fields_basic):
        try:
            out = subprocess.check_output(
                ["nvidia-smi", f"--query-gpu={fields}",
                 "--format=csv,noheader,nounits"],
                text=True, timeout=5,
            )
            break
        except Exception:
            continue
    if not out:
        return []
    gpus = []
    for line in out.strip().splitlines():
        parts = [p.strip() for p in line.split(",")]
        if len(parts) < 9:
            continue
        name = parts[1]
        tag = "SXM2" if "SXM2" in name else "PCIe" if "PCIE" in name or "PCIe" in name else "GPU"
        gpu = {
            "idx": parts[0],
            "tag": tag,
            "temp": int(parts[2]),
            "power": float(parts[3]),
            "power_limit": float(parts[4]),
            "util": int(parts[5]),
            "vram_used": int(parts[6]),
            "vram_total": int(parts[7]),
            "bus_id": parts[8],
        }
        # Power limit range (min/max always present, default only with full query)
        if len(parts) >= 11:
            try:
                gpu["pl_min"] = float(parts[9])
                gpu["pl_max"] = float(parts[10])
                gpu["pl_default"] = float(parts[11]) if len(parts) >= 12 else float(parts[4])
            except (ValueError, IndexError):
                pass
        gpus.append(gpu)
    return gpus


def query_gpu_topology(gpus):
    """Detect NVLink baseboard groups.

    Tries nvidia-smi topo first. Falls back to grouping SXM2 GPUs by
    adjacent PCI bus numbers (SXM2 baseboard pairs always occupy
    consecutive PCI buses).

    Returns dict mapping nvidia-smi GPU index (str) -> board group (int).
    Only SXM2 GPUs are grouped; PCIe GPUs are excluded.
    """
    # --- Try nvidia-smi topo first ---
    try:
        out = subprocess.check_output(
            ["nvidia-smi", "topo", "-m"],
            text=True,
            timeout=5,
        )
        lines = out.strip().splitlines()
        gpu_rows = []
        for line in lines:
            parts = line.split()
            if len(parts) >= 2 and parts[0].startswith("GPU") and "X" in parts[1:]:
                try:
                    idx = int(parts[0][3:])
                except ValueError:
                    continue
                gpu_rows.append((idx, parts[1:]))

        if gpu_rows:
            n = len(gpu_rows)
            parent = list(range(n))

            def find(x):
                while parent[x] != x:
                    parent[x] = parent[parent[x]]
                    x = parent[x]
                return x

            def union(x, y):
                px, py = find(x), find(y)
                if px != py:
                    parent[px] = py

            for i in range(n):
                conns = gpu_rows[i][1]
                for j in range(min(n, len(conns))):
                    if conns[j].startswith("NV"):
                        union(i, j)

            result = {}
            group_ids = {}
            next_id = 0
            for i in range(n):
                root = find(i)
                if root not in group_ids:
                    group_ids[root] = next_id
                    next_id += 1
                result[str(gpu_rows[i][0])] = group_ids[root]
            if result:
                return result
    except Exception:
        pass

    # --- Fallback: group SXM2 GPUs by PCI bus topology ---
    sxm2 = [(gpu["idx"], bus_id_to_bus_number(gpu["bus_id"]))
            for gpu in gpus if gpu["tag"] == "SXM2"]
    sxm2 = [(idx, bus) for idx, bus in sxm2 if bus is not None]
    if not sxm2:
        return {}

    sxm2.sort(key=lambda x: x[1])

    if len(sxm2) <= 4:
        # Single board — all in one group
        return {idx: 0 for idx, _ in sxm2}

    # Calculate gaps between consecutive GPUs
    gaps = [(sxm2[i + 1][1] - sxm2[i][1], i) for i in range(len(sxm2) - 1)]

    # Number of boards (4 GPUs per SXM2 baseboard)
    num_boards = max(1, len(sxm2) // 4)
    num_splits = num_boards - 1

    # Find the largest gaps as board boundaries
    gaps_sorted = sorted(gaps, key=lambda x: x[0], reverse=True)
    split_positions = sorted(g[1] for g in gaps_sorted[:num_splits])

    result = {}
    group_id = 0
    for i, (idx_str, bus) in enumerate(sxm2):
        result[idx_str] = group_id
        if group_id < len(split_positions) and i == split_positions[group_id]:
            group_id += 1

    return result


def query_system_ram():
    """Get system RAM usage via Windows GlobalMemoryStatusEx API."""
    try:
        class MEMORYSTATUSEX(ctypes.Structure):
            _fields_ = [
                ("dwLength", ctypes.c_ulong),
                ("dwMemoryLoad", ctypes.c_ulong),
                ("ullTotalPhys", ctypes.c_ulonglong),
                ("ullAvailPhys", ctypes.c_ulonglong),
                ("ullTotalPageFile", ctypes.c_ulonglong),
                ("ullAvailPageFile", ctypes.c_ulonglong),
                ("ullTotalVirtual", ctypes.c_ulonglong),
                ("ullAvailVirtual", ctypes.c_ulonglong),
                ("ullAvailExtendedVirtual", ctypes.c_ulonglong),
            ]
        stat = MEMORYSTATUSEX()
        stat.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
        ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
        total = stat.ullTotalPhys / (1024 ** 3)
        used = (stat.ullTotalPhys - stat.ullAvailPhys) / (1024 ** 3)
        return {"total_gb": total, "used_gb": used, "pct": stat.dwMemoryLoad}
    except Exception:
        return None


def query_loaded_models():
    """Query Ollama API for loaded models with RAM/VRAM split."""
    try:
        req = urllib.request.Request("http://localhost:11434/api/ps", method="GET")
        with urllib.request.urlopen(req, timeout=0.5) as resp:
            data = json.loads(resp.read().decode("utf-8"))
        models = []
        for m in data.get("models", []):
            name = m.get("name", "unknown")
            short = name.split(":")[0].split("/")[-1]
            if len(short) > 18:
                short = short[:16] + ".."
            size_gb = m.get("size", 0) / (1024 ** 3)
            vram_gb = m.get("size_vram", 0) / (1024 ** 3)
            ram_gb = max(0, size_gb - vram_gb)
            models.append({"name": short, "size_gb": size_gb,
                           "vram_gb": vram_gb, "ram_gb": ram_gb})
        return models
    except Exception:
        return []


def query_llamacpp_model():
    """Check llama.cpp server on port 8080 for loaded model."""
    try:
        req = urllib.request.Request("http://localhost:8080/props", method="GET")
        with urllib.request.urlopen(req, timeout=0.5) as resp:
            data = json.loads(resp.read().decode("utf-8"))
        model = data.get("default_generation_settings", {}).get("model", "")
        if model:
            short = model.split("/")[-1].split(".gguf")[0]
            if len(short) > 22:
                short = short[:20] + ".."
            return short
        return None
    except Exception:
        return None


def bus_id_to_bus_number(bus_id):
    """Convert nvidia-smi bus ID '00000000:3B:00.0' to decimal bus number (59)."""
    try:
        hex_bus = bus_id.split(":")[1]
        return int(hex_bus, 16)
    except (IndexError, ValueError):
        return None


def set_power_limit(gpu_idx, watts):
    """Set GPU power limit via nvidia-smi. Requires Administrator."""
    try:
        subprocess.check_output(
            ["nvidia-smi", "-i", str(gpu_idx), "-pl", str(int(watts))],
            text=True, timeout=10, stderr=subprocess.STDOUT,
        )
        return True, None
    except subprocess.CalledProcessError as e:
        msg = (e.output or "").strip()
        if any(w in msg.lower() for w in ("denied", "administrator", "access",
                                           "insufficient")):
            return False, "Run as Administrator to set power limits"
        return False, msg or "Failed to set power limit"
    except Exception as e:
        return False, str(e)


def temp_color(t):
    if t < 60:
        return "#4ec94e"
    if t < 80:
        return "#e6c84c"
    return "#e64c4c"


class GpuMonitor(tk.Tk):
    def __init__(self):
        super().__init__()
        self.overrideredirect(True)
        self.attributes("-topmost", True)
        self.configure(bg=BG)

        self._dx = 0
        self._dy = 0
        self.bind("<ButtonPress-1>", self._start_drag)
        self.bind("<B1-Motion>", self._on_drag)
        self.bind("<ButtonPress-3>", self._context_menu)

        self.rows = {}              # bus_id -> row widget dict
        self._gpu_order = []        # ordered bus_ids for display
        self._board_groups = {}     # nvidia_idx (str) -> group_id (int)
        self._power_ranges = {}     # bus_id -> {idx, pl_min, pl_max, pl_default}
        self._refresh_after_id = None
        self._boner_after_id = None
        self._status_after_id = None

        self._build_ui()
        self._refresh()

        # Position top-right
        self.update_idletasks()
        x = self.winfo_screenwidth() - self.winfo_width() - 16
        self.geometry(f"+{x}+16")

    # ------------------------------------------------------------------
    # UI construction
    # ------------------------------------------------------------------

    def _draw_flame(self, parent):
        c = tk.Canvas(parent, width=14, height=18, bg=BG, highlightthickness=0)
        c.create_polygon(7, 0, 2, 12, 4, 18, 10, 18, 12, 12, fill="#e63e3e", outline="")
        c.create_polygon(7, 5, 4, 13, 5, 17, 9, 17, 10, 13, fill="#f5943d", outline="")
        c.create_polygon(7, 9, 5, 14, 6, 17, 8, 17, 9, 14, fill="#f5d43d", outline="")
        return c

    def _build_ui(self):
        # Title bar
        title_frame = tk.Frame(self, bg=BG)
        title_frame.pack(fill="x", padx=6, pady=(4, 2))
        self._draw_flame(title_frame).pack(side="left", padx=(0, 4))
        tk.Label(
            title_frame, text="Fart Machine GPU Meter", bg=BG, fg="#e64c4c",
            font=("Consolas", 10, "bold"),
        ).pack(side="left")
        self._draw_flame(title_frame).pack(side="left", padx=(4, 0))
        tk.Frame(self, bg="#444", height=1).pack(fill="x", padx=6, pady=(2, 2))

        gpus = query_gpus()
        self._board_groups = query_gpu_topology(gpus)

        if not gpus:
            tk.Label(self, text="No GPUs found", bg=BG, fg="#e64c4c",
                     font=("Consolas", 9)).pack(padx=8, pady=4)

        # Partition active GPUs: SXM2 by board group, then ungrouped / PCIe
        self._gpu_order = []
        board_gpu_groups = {}   # group_id -> [gpu_dict, ...]
        ungrouped = []

        for gpu in gpus:
            # Cache power limit range for context menu
            if "pl_min" in gpu:
                self._power_ranges[gpu["bus_id"]] = {
                    "idx": gpu["idx"],
                    "pl_min": gpu["pl_min"],
                    "pl_max": gpu["pl_max"],
                    "pl_default": gpu["pl_default"],
                }
            group = self._board_groups.get(gpu["idx"])
            if group is not None and gpu["tag"] == "SXM2":
                board_gpu_groups.setdefault(group, []).append(gpu)
            else:
                ungrouped.append(gpu)

        # --- Render board-grouped SXM2 GPUs ---
        for group_id in sorted(board_gpu_groups.keys()):
            group_gpus = board_gpu_groups[group_id]
            board_color = BOARD_COLORS[group_id % len(BOARD_COLORS)]

            # Board header
            hdr = tk.Frame(self, bg=BG)
            hdr.pack(fill="x", padx=6, pady=(4, 1))
            tk.Canvas(hdr, width=8, height=8, bg=board_color,
                      highlightthickness=0).pack(side="left", padx=(0, 4))
            tk.Label(hdr, text=f"Board {group_id + 1}", bg=BG, fg=FG_DIM,
                     font=("Consolas", 8)).pack(side="left")

            for gpu in group_gpus:
                self._gpu_order.append(gpu["bus_id"])
                self._create_gpu_row(gpu["bus_id"], gpu["idx"], gpu["tag"],
                                     board_color=board_color)

        # --- Render ungrouped / PCIe GPUs ---
        if ungrouped:
            if board_gpu_groups:
                tk.Frame(self, bg="#333", height=1).pack(fill="x", padx=6, pady=(4, 1))
            for gpu in ungrouped:
                self._gpu_order.append(gpu["bus_id"])
                self._create_gpu_row(gpu["bus_id"], gpu["idx"], gpu["tag"])

        # Model info section
        tk.Frame(self, bg="#444", height=1).pack(fill="x", padx=6, pady=(4, 2))
        self._model_frame = tk.Frame(self, bg=BG)
        self._model_frame.pack(fill="x", padx=6, pady=(0, 2))
        self._model_lbl = tk.Label(
            self._model_frame, text="Models: checking...", bg=BG, fg=FG_MODEL,
            font=("Consolas", 8), anchor="w", justify="left",
        )
        self._model_lbl.pack(anchor="w")

        # System RAM bar
        ram_row = tk.Frame(self._model_frame, bg=BG)
        ram_row.pack(fill="x", pady=(2, 0))
        self._ram_lbl = tk.Label(ram_row, text="RAM:", bg=BG, fg=FG_DIM,
                                 font=("Consolas", 8), anchor="w")
        self._ram_lbl.pack(side="left")
        self._ram_canvas = tk.Canvas(ram_row, width=RAM_BAR_W, height=RAM_BAR_H,
                                     bg="#333", highlightthickness=0)
        self._ram_canvas.pack(side="left", padx=(4, 4))
        self._ram_bar = self._ram_canvas.create_rectangle(
            0, 0, 0, RAM_BAR_H, fill="#7a5abf", outline="")
        self._ram_pct_lbl = tk.Label(ram_row, text="", bg=BG, fg=FG_DIM,
                                     font=("Consolas", 8))

        # Status message (shown on power limit feedback, hidden by default)
        self._status_lbl = tk.Label(self, text="", bg=BG, fg="#e6c84c",
                                    font=("Consolas", 8))

        # Boner Zone alert (hidden by default)
        self._boner_frame = tk.Frame(self, bg=BG)
        self._boner_frame.pack(fill="x", padx=6, pady=(4, 4))
        self._boner_sep = tk.Frame(self._boner_frame, bg="#444", height=1)
        self._boner_sep.pack(fill="x", pady=(0, 4))
        self._boner_lbl = tk.Label(
            self._boner_frame, text="BONER ZONE", bg=BG, fg="#e64c4c",
            font=("Consolas", 12, "bold"),
        )
        self._boner_lbl.pack()
        self._boner_frame.pack_forget()
        self._boner_visible = False
        self._boner_flash_on = True

    def _create_gpu_row(self, bus_id, idx, tag, board_color=None):
        """Create one GPU row with telemetry and optional board accent."""
        row_frame = tk.Frame(self, bg=BG)
        row_frame.pack(fill="x", padx=6, pady=1)

        # Board color accent stripe (thin left bar)
        if board_color:
            tk.Frame(row_frame, bg=board_color, width=3).pack(
                side="left", fill="y", padx=(0, 3))

        # GPU index + tag
        id_lbl = tk.Label(
            row_frame, text=f"{idx} {tag:5s}",
            bg=BG, fg=FG_DIM,
            font=("Consolas", 9), anchor="w", width=8,
        )
        id_lbl.pack(side="left")

        # Temp
        temp_lbl = tk.Label(row_frame, text="", bg=BG, fg=FG,
                            font=("Consolas", 9), anchor="e", width=4)
        temp_lbl.pack(side="left")

        # Temp bar
        canvas = tk.Canvas(row_frame, width=BAR_W, height=BAR_H,
                           bg="#333", highlightthickness=0)
        canvas.pack(side="left", padx=(4, 4))
        bar_rect = canvas.create_rectangle(0, 0, 0, BAR_H, fill="#4ec94e", outline="")

        # VRAM label
        vram_lbl = tk.Label(row_frame, text="", bg=BG, fg=FG_DIM,
                            font=("Consolas", 8), anchor="e", width=11)
        vram_lbl.pack(side="left")

        # VRAM bar
        vram_canvas = tk.Canvas(row_frame, width=VRAM_BAR_W, height=VRAM_BAR_H,
                                bg="#333", highlightthickness=0)
        vram_canvas.pack(side="left", padx=(2, 4))
        vram_bar = vram_canvas.create_rectangle(0, 0, 0, VRAM_BAR_H,
                                                fill="#3a7ebf", outline="")

        # Power draw (clickable — cycles power limit: min -> default -> max)
        pwr_lbl = tk.Label(row_frame, text="", bg=BG, fg=FG,
                           font=("Consolas", 9), anchor="e", width=10,
                           cursor="hand2")
        pwr_lbl.pack(side="left")
        pwr_lbl.bind("<Button-1>", lambda e, bid=bus_id: self._cycle_power_limit(bid))

        # Utilization
        util_lbl = tk.Label(row_frame, text="", bg=BG, fg=FG,
                            font=("Consolas", 9), anchor="e", width=5)
        util_lbl.pack(side="left")

        self.rows[bus_id] = {
            "frame": row_frame, "id_lbl": id_lbl, "tag": tag,
            "temp_lbl": temp_lbl, "canvas": canvas, "bar": bar_rect,
            "vram_lbl": vram_lbl, "vram_canvas": vram_canvas, "vram_bar": vram_bar,
            "pwr_lbl": pwr_lbl, "util_lbl": util_lbl, "bus_id": bus_id,
            "board_color": board_color,
        }

    # ------------------------------------------------------------------
    # Power limit cycling
    # ------------------------------------------------------------------

    def _cycle_power_limit(self, bus_id):
        """Click handler: cycle GPU power limit through min -> default -> max."""
        pr = self._power_ranges.get(bus_id)
        if not pr:
            self._show_status("Power limit range not available", error=True)
            return
        gpu_idx = pr["idx"]
        pl_min = int(pr["pl_min"])
        pl_max = int(pr["pl_max"])
        pl_def = int(pr["pl_default"])

        # Get current limit
        gpus = query_gpus()
        cur = None
        for g in gpus:
            if g["bus_id"] == bus_id:
                cur = int(g["power_limit"])
                break
        if cur is None:
            return

        # Cycle: min -> default -> max -> min
        presets = sorted(set([pl_min, pl_def, pl_max]))
        # Find next preset after current
        target = presets[0]
        for p in presets:
            if p > cur + 2:  # +2 for rounding tolerance
                target = p
                break
        else:
            target = presets[0]  # wrap around to min

        ok, err = set_power_limit(gpu_idx, target)
        if ok:
            label = "(min)" if target == pl_min else "(max)" if target == pl_max else "(default)"
            self._show_status(f"GPU {gpu_idx} -> {target}W {label}")
        else:
            self._show_status(err or "Failed", error=True)

    def _show_status(self, msg, error=False):
        self._status_lbl.config(text=msg, fg="#e64c4c" if error else "#e6c84c")
        self._status_lbl.pack(fill="x", padx=6, pady=(0, 4))
        if self._status_after_id:
            self.after_cancel(self._status_after_id)
        self._status_after_id = self.after(5000, self._hide_status)

    def _hide_status(self):
        self._status_lbl.pack_forget()
        self._status_after_id = None

    # ------------------------------------------------------------------
    # Refresh loop
    # ------------------------------------------------------------------

    def _refresh(self):
        try:
            gpus = query_gpus()
            any_sxm2_hot = False
            total_vram_used = 0
            total_vram = 0

            gpu_by_bus = {gpu["bus_id"]: gpu for gpu in gpus}

            for bus_id, row in self.rows.items():
                gpu = gpu_by_bus.get(bus_id)
                if not gpu:
                    continue

                t = gpu["temp"]
                color = temp_color(t)
                hot = row["tag"] == "SXM2" and t >= 79
                if hot:
                    any_sxm2_hot = True

                vram_used = gpu.get("vram_used", 0)
                vram_total = gpu.get("vram_total", 32768)
                total_vram_used += vram_used
                total_vram += vram_total
                vram_pct = vram_used / vram_total if vram_total > 0 else 0
                vram_gb_used = vram_used / 1024
                vram_gb_total = vram_total / 1024

                row_bg = "#4a1010" if hot else BG
                row["frame"].config(bg=row_bg)
                row["id_lbl"].config(bg=row_bg, fg="#ff6666" if hot else FG_DIM)
                row["temp_lbl"].config(text=f"{t}\u00b0C", bg=row_bg,
                                       fg="#ff4444" if hot else FG)
                row["canvas"].config(bg="#552222" if hot else "#333")
                row["pwr_lbl"].config(
                    text=f"{gpu['power']:.0f}/{gpu['power_limit']:.0f}W",
                    bg=row_bg, fg="#ff6666" if hot else FG)
                row["util_lbl"].config(text=f"{gpu['util']}%",
                                       bg=row_bg, fg="#ff6666" if hot else FG)

                bar_w = max(1, int((t / 100) * BAR_W))
                row["canvas"].coords(row["bar"], 0, 0, bar_w, BAR_H)
                row["canvas"].itemconfig(row["bar"], fill=color)

                vram_color = ("#3a7ebf" if vram_pct < 0.85 else
                              "#e6c84c" if vram_pct < 0.95 else "#e64c4c")
                vram_bar_w = max(1, int(vram_pct * VRAM_BAR_W))
                row["vram_lbl"].config(
                    text=f"{vram_gb_used:.0f}/{vram_gb_total:.0f}GB",
                    bg=row_bg, fg="#ff6666" if hot else FG_DIM)
                row["vram_canvas"].config(bg="#552222" if hot else "#333")
                row["vram_canvas"].coords(row["vram_bar"], 0, 0,
                                          vram_bar_w, VRAM_BAR_H)
                row["vram_canvas"].itemconfig(row["vram_bar"], fill=vram_color)

            # Model info
            model_lines = []
            ollama_models = query_loaded_models()
            for m in ollama_models:
                if m["ram_gb"] > 0.1 and m["vram_gb"] > 0.1:
                    # Split across RAM + VRAM (MoE / partial offload)
                    model_lines.append(
                        f"Ollama: {m['name']} "
                        f"({m['vram_gb']:.1f}GB VRAM + {m['ram_gb']:.1f}GB RAM)")
                elif m["ram_gb"] > 0.1:
                    # Fully in RAM (CPU mode)
                    model_lines.append(
                        f"Ollama: {m['name']} ({m['ram_gb']:.1f}GB RAM)")
                else:
                    # Fully in VRAM
                    model_lines.append(
                        f"Ollama: {m['name']} ({m['vram_gb']:.1f}GB VRAM)")
            llama_model = query_llamacpp_model()
            if llama_model:
                model_lines.append(f"llama.cpp: {llama_model}")
            if not model_lines:
                model_lines.append("No models loaded")
            if total_vram > 0:
                total_gb = total_vram_used / 1024
                total_max_gb = total_vram / 1024
                model_lines.append(
                    f"Total VRAM: {total_gb:.0f}/{total_max_gb:.0f} GB "
                    f"({total_gb / total_max_gb * 100:.0f}%)")
            self._model_lbl.config(text="\n".join(model_lines))

            # System RAM bar
            ram = query_system_ram()
            if ram:
                used, total_ram, pct = ram["used_gb"], ram["total_gb"], ram["pct"]
                self._ram_lbl.config(text=f"RAM: {used:.0f}/{total_ram:.0f}GB")
                bar_w = max(1, int((pct / 100) * RAM_BAR_W))
                ram_color = "#7a5abf" if pct < 75 else "#e6c84c" if pct < 90 else "#e64c4c"
                self._ram_canvas.coords(self._ram_bar, 0, 0, bar_w, RAM_BAR_H)
                self._ram_canvas.itemconfig(self._ram_bar, fill=ram_color)
                self._ram_pct_lbl.config(text=f"{pct}%")
                self._ram_pct_lbl.pack(side="left")

            # Boner Zone
            if any_sxm2_hot and not self._boner_visible:
                self._boner_frame.pack(fill="x", padx=6, pady=(4, 4))
                self._boner_visible = True
                self._flash_boner()
            elif not any_sxm2_hot and self._boner_visible:
                self._boner_frame.pack_forget()
                self._boner_visible = False
        except Exception:
            pass
        finally:
            self._refresh_after_id = self.after(REFRESH_MS, self._refresh)

    def _flash_boner(self):
        if not self._boner_visible:
            return
        self._boner_flash_on = not self._boner_flash_on
        self._boner_lbl.config(fg="#e64c4c" if self._boner_flash_on else BG)
        self._boner_after_id = self.after(500, self._flash_boner)

    # ------------------------------------------------------------------
    # Window interaction
    # ------------------------------------------------------------------

    def _start_drag(self, event):
        self._dx = event.x
        self._dy = event.y

    def _on_drag(self, event):
        x = self.winfo_x() + event.x - self._dx
        y = self.winfo_y() + event.y - self._dy
        self.geometry(f"+{x}+{y}")

    def _context_menu(self, event):
        menu = tk.Menu(self, tearoff=0)

        # Power Limits submenu
        if self._power_ranges:
            pl_menu = tk.Menu(menu, tearoff=0)
            # Get current limits for checkmarks
            current_gpus = query_gpus()
            current_pl = {g["bus_id"]: g["power_limit"] for g in current_gpus}

            for bus_id in self._gpu_order:
                pr = self._power_ranges.get(bus_id)
                row = self.rows.get(bus_id)
                if not pr or not row:
                    continue
                idx_text = row["id_lbl"].cget("text").strip()
                gpu_menu = tk.Menu(pl_menu, tearoff=0)

                # Generate presets in 25W steps from min to max
                pl_min = int(pr["pl_min"])
                pl_max = int(pr["pl_max"])
                pl_def = int(pr["pl_default"])
                cur = int(current_pl.get(bus_id, 0))
                gpu_idx = pr["idx"]

                presets = list(range(pl_min, pl_max + 1, 25))
                if pl_def not in presets:
                    presets.append(pl_def)
                if pl_max not in presets:
                    presets.append(pl_max)
                presets = sorted(set(presets))

                for w in presets:
                    suffix = ""
                    if w == pl_def:
                        suffix = " (default)"
                    elif w == pl_min:
                        suffix = " (min)"
                    elif w == pl_max:
                        suffix = " (max)"
                    check = abs(w - cur) < 2
                    gpu_menu.add_command(
                        label=f"{'> ' if check else '   '}{w}W{suffix}",
                        command=lambda idx=gpu_idx, watts=w: self._set_pl(idx, watts))

                pl_menu.add_cascade(label=idx_text, menu=gpu_menu)

            # "Set All" presets
            if len(self._power_ranges) > 1:
                pl_menu.add_separator()
                all_menu = tk.Menu(pl_menu, tearoff=0)
                # Use the most common default as reference
                defaults = [int(pr["pl_default"]) for pr in self._power_ranges.values()]
                mins = [int(pr["pl_min"]) for pr in self._power_ranges.values()]
                common_def = max(set(defaults), key=defaults.count)
                common_min = max(set(mins), key=mins.count)
                for w in sorted({common_min, common_def}):
                    suffix = " (default)" if w == common_def else " (min)" if w == common_min else ""
                    all_menu.add_command(
                        label=f"{w}W{suffix}",
                        command=lambda watts=w: self._set_pl_all(watts))
                pl_menu.add_cascade(label="Set All", menu=all_menu)

            menu.add_cascade(label="Power Limits", menu=pl_menu)
            menu.add_separator()

        menu.add_command(label="Close", command=self.destroy)
        menu.tk_popup(event.x_root, event.y_root)

    def _set_pl(self, gpu_idx, watts):
        """Set power limit for a single GPU."""
        ok, err = set_power_limit(gpu_idx, watts)
        if ok:
            self._show_status(f"GPU {gpu_idx} -> {watts}W")
        else:
            self._show_status(err or "Failed to set power limit", error=True)

    def _set_pl_all(self, watts):
        """Set power limit for all active GPUs."""
        fails = 0
        for bus_id, pr in self._power_ranges.items():
            clamped = max(int(pr["pl_min"]), min(watts, int(pr["pl_max"])))
            ok, _ = set_power_limit(pr["idx"], clamped)
            if not ok:
                fails += 1
        if fails:
            self._show_status(f"Set power limits ({fails} failed)", error=True)
        else:
            self._show_status(f"All GPUs -> {watts}W")


if __name__ == "__main__":
    GpuMonitor().mainloop()