diff --git a/docs/images/cfg.svg b/docs/images/cfg.svg new file mode 100644 index 0000000..0a18970 --- /dev/null +++ b/docs/images/cfg.svg @@ -0,0 +1 @@ +FFTTentry %cmp7 = icmp sgt i32 %n, 0 br i1 %cmp7, label %for.body, label %for.cond.cleanupfor.cond.cleanup %s.0.lcssa = phi i32 [ 0, %entry ], [ %s.1, %for.body ] ret i32 %s.0.lcssafor.body %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ] %s.08 = phi i32 [ %s.1, %for.body ], [ 0, %entry ] %and = and i32 %i.09, 1 %tobool.not = icmp eq i32 %and, 0 %0 = sub nsw i32 0, %i.09 %s.1.p = select i1 %tobool.not, i32 %0, i32 %i.09 %s.1 = add i32 %s.1.p, %s.08 %inc = add nuw nsw i32 %i.09, 1 %exitcond.not = icmp eq i32 %inc, %n br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !10 diff --git a/ruff.toml b/ruff.toml index a188dc0..9cb5280 100644 --- a/ruff.toml +++ b/ruff.toml @@ -48,4 +48,7 @@ ignore = ["S603", "S607"] # off for the same reason: several fixtures are verbatim `opt` output, one line # of which is 111 characters, and a fixture that has been reflowed to fit a # column limit is no longer evidence of anything. -"toolkit/tests/*" = ["S101", "S105", "S108", "E501"] +# S314 is the XML parser one. What gets parsed here is the SVG this package +# generated two lines earlier, and the check is that it is well formed, which +# is exactly the thing a hardened parser would refuse to tell us. +"toolkit/tests/*" = ["S101", "S105", "S108", "S314", "E501"] diff --git a/toolkit/README.md b/toolkit/README.md index 4f4d032..d8c8f7e 100644 --- a/toolkit/README.md +++ b/toolkit/README.md @@ -226,6 +226,31 @@ The wrong options get a real explanation too. A reader who picked the second one Same rules as the tape: no JavaScript, the reveal is a `
`, the highlight on what you picked is a radio button and a sibling selector, and the answer is named in a sentence as well as tinted so a renderer that drops the stylesheet still tells you which one it was. Printing a gate in a terminal shows the question, the options, the answer and every explanation, because there is nowhere in plain text to hide something behind a click and pretending otherwise would make it a puzzle instead of a lesson. +## Looking at the graph + +`opt` has printers for the control flow graph, the dominator and post dominator trees, the data dependence graph and the call graph. They are in a release build, and they write Graphviz `.dot` into whatever directory you are standing in. `irx.cfg` runs one, reads what it wrote, and draws it. + +```python +m = irx.Module.from_c(""" +int f(int n) { + int s = 0; + for (int i = 0; i < n; i++) { if (i & 1) s += i; else s -= i; } + return s; +} +""", opt="-O1") +irx.cfg(m) +``` + +![The control flow graph of a loop, drawn from opt's own dot output](../docs/images/cfg.svg) + +`irx.dom`, `irx.ddg` and `irx.callgraph` are the same thing with a different printer behind them, and `irx.graph(m, "post-dom")` reaches the rest. A module with more than one function is an error naming the functions it has, rather than a picture of whichever one came first. + +Nothing here works out what the successors of a block are. That is the whole point: the graph is LLVM's, so when it is surprising the thing to argue with is LLVM. What this module does is parse the `.dot`, throw away the node names, which are pointer addresses and differ between two runs of the same program, and lay the result out. + +**The SelectionDAG viewers are not here, and cannot be.** `-view-isel-dags` and its family are declared inside `#ifndef NDEBUG`, and the `#else` branch replaces them with `static const bool ... = false`, so on a release toolchain those flags do not exist at all (`llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp:147-189@llvmorg-23.1.0`). A reader who installed LLVM from a package manager cannot have them, so no lesson may depend on them. Of the graphs a release `opt` will give you, the data dependence graph is the one that is genuinely a DAG: it collapses each cycle into a pi-block, which is what makes the rest of it acyclic. + +**No Graphviz.** `dot` is not installed on every machine this has to run on, so the layout is here instead: rank each node one below its furthest predecessor, break the cycles first with a depth first walk, and route the two awkward cases, back edges and edges that skip a row, around the margin rather than straight through whatever box is in the way. The output is one inline `` with no script in it, which survives being saved into a notebook and served as static HTML. + ## When a tool fails The default subprocess failure is `CalledProcessError: returned non-zero exit status 1`, which tells a reader who has never run `opt` before absolutely nothing. `irx` raises `ToolError` instead, with the command, the real stderr, and where possible a sentence about what to do: diff --git a/toolkit/irx/__init__.py b/toolkit/irx/__init__.py index ae83e80..a76d01e 100644 --- a/toolkit/irx/__init__.py +++ b/toolkit/irx/__init__.py @@ -23,9 +23,10 @@ from __future__ import annotations from . import env as _env +from . import graphs, pipeline, plugin, predict, verify from . import magic as _magic -from . import pipeline, plugin, predict, verify from .env import Env, cpu_count, describe, detect +from .graphs import Edge, Graph, GraphError, Node, callgraph, cfg, ddg, dom, graph from .ir import Diff, Module, has_pass, highlight, passes from .magic import load_ipython_extension from .pipeline import Step, Tape, tape @@ -41,9 +42,13 @@ __all__ = [ "AliveError", "Diff", + "Edge", "Env", "Gate", + "Graph", + "GraphError", "Module", + "Node", "Plugin", "Report", "Result", @@ -55,12 +60,18 @@ "Verdict", "alive", "bootstrap", + "callgraph", + "cfg", "compile_c", "cpu_count", "current", + "ddg", "describe", "detect", + "dom", "gate", + "graph", + "graphs", "has_pass", "highlight", "load_ipython_extension", diff --git a/toolkit/irx/graphs.py b/toolkit/irx/graphs.py new file mode 100644 index 0000000..f6a8e77 --- /dev/null +++ b/toolkit/irx/graphs.py @@ -0,0 +1,700 @@ +"""LLVM's own graphs, drawn where a reader can see them. + +Every graph in here comes out of LLVM. Nothing in this file works out what the +successors of a block are, or which block dominates which. `opt` has printers +for all of it, they are in a release build, and they write Graphviz `.dot` to +the working directory. This module runs one of them, reads what it wrote, and +turns it into a picture. When the graph is wrong it is LLVM that is wrong, which +is the only arrangement worth having in a book that teaches people to go and +look. + +Two things are worth saying out loud about the choice of graphs. + +The SelectionDAG viewers, `-view-isel-dags` and the rest of that family, are not +here. They only exist in an assertions build: every one of those options is +declared inside `#ifndef NDEBUG`, and the `#else` branch replaces them with +`static const bool ... = false`, so on a release toolchain the flag is not +merely off, it is not a flag +(`llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp:147-189@llvmorg-23.1.0`). +A reader who installed LLVM from a package manager cannot have them, so a lesson +cannot use them. What a release `opt` will give you is here instead, and the +data dependence graph is the one of these that is genuinely a DAG. + +There is no Graphviz here either. `dot` is not installed on every machine this +course has to run on, and shelling out to it would mean a lesson that renders on +the author's laptop and prints a stack trace in Colab. The layout below is +ordinary layered drawing, about eighty lines of it, and the output is one inline +SVG element with no script in it, so it survives being saved into a notebook and +served as static HTML. +""" + +from __future__ import annotations + +import hashlib +import html +import re +import tempfile +from dataclasses import dataclass, field +from pathlib import Path + +from .ir import COLOUR, KEYWORDS, TOKEN, Module +from .proc import run + +# The pass name, and the shape of the file it leaves behind. `opt` prints +# `Writing ''...` as it goes, but matching on a glob is steadier than +# parsing a progress message that has changed wording before. +KINDS = { + "cfg": ("dot-cfg", "*.dot", "the control flow graph, with each block's instructions"), + "cfg-only": ("dot-cfg-only", "*.dot", "the control flow graph, blocks named only"), + "dom": ("dot-dom", "dom.*.dot", "the dominator tree"), + "post-dom": ("dot-post-dom", "postdom.*.dot", "the post dominator tree"), + "ddg": ("dot-ddg", "ddg.*.dot", "the data dependence graph"), + "callgraph": ("dot-callgraph", "*.callgraph.dot", "who calls whom"), +} + + +# -- reading what LLVM wrote -------------------------------------------------- + +NODE = re.compile(r"^\s*(Node0x[0-9a-fA-F]+)\s*\[(?P.*)\];\s*$") +EDGE = re.compile( + r"^\s*(Node0x[0-9a-fA-F]+)(?::(?P\w+))?\s*->\s*" + r"(Node0x[0-9a-fA-F]+)(?::\w+)?\s*(?:\[(?P.*)\])?;\s*$" +) +LABEL = re.compile(r'label\s*=\s*"((?:[^"\\]|\\.)*)"') +GRAPH_LABEL = re.compile(r'^\s*label\s*=\s*"((?:[^"\\]|\\.)*)"\s*;\s*$') +PORT = re.compile(r"^<(?P\w+)>(?P.*)$", re.S) + + +def _unescape(raw: str) -> str: + """Graphviz label escapes, flattened to plain text. + + `\\l` and `\\n` are both line breaks as far as this is concerned; the + difference between them is justification, and the boxes here are left + aligned regardless. + """ + out: list[str] = [] + index = 0 + while index < len(raw): + char = raw[index] + if char == "\\" and index + 1 < len(raw): + following = raw[index + 1] + out.append("\n" if following in "lnr" else following) + index += 2 + else: + out.append(char) + index += 1 + return "".join(out) + + +def _split(raw: str) -> list[str]: + """Split a record label on the bars that are structure, not text. + + A bar inside braces belongs to a nested record, and a backslashed bar is a + bar somebody wanted to print. Neither is a field separator, which is the + entire reason this is not `raw.split("|")`. + """ + fields: list[str] = [] + depth = 0 + start = 0 + index = 0 + while index < len(raw): + char = raw[index] + if char == "\\": + index += 2 + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + elif char == "|" and depth == 0: + fields.append(raw[start:index]) + start = index + 1 + index += 1 + fields.append(raw[start:]) + return fields + + +@dataclass +class Node: + """One box. `title` is the block or function name, `lines` is what is in it.""" + + key: str + title: str + lines: list[str] = field(default_factory=list) + ports: dict[str, str] = field(default_factory=dict) + + @property + def name(self) -> str: + return self.title or self.key + + +@dataclass +class Edge: + src: str + dst: str + label: str = "" + back: bool = False + + +def _node(key: str, attrs: str) -> Node: + match = LABEL.search(attrs) + if not match: + return Node(key=key, title="") + raw = match.group(1).strip() + if raw.startswith("{") and raw.endswith("}"): + raw = raw[1:-1] + fields = _split(raw) + + ports: dict[str, str] = {} + tail = fields[-1].strip() + if len(fields) > 1 and tail.startswith("{") and tail.endswith("}"): + for piece in _split(tail[1:-1]): + port = PORT.match(piece.strip()) + if port: + ports[port.group("name")] = _unescape(port.group("text")).strip() + fields = fields[:-1] + + title = "" + if len(fields) > 1: + title = _unescape(fields[0]).strip().rstrip(":") + fields = fields[1:] + + lines: list[str] = [] + for piece in fields: + lines += [line for line in _unescape(piece).split("\n") if line.strip()] + # A single field graph, the call graph being the one that matters, has the + # name in the body rather than in a title field. Promote it, so every node + # has something to be called. + if not title and len(lines) == 1: + title, lines = lines[0].strip(), [] + return Node(key=key, title=title, lines=lines, ports=ports) + + +def parse(dot: str) -> tuple[str, list[Node], list[Edge]]: + """Graphviz in, a caption and a graph out. Pointer names never leave here. + + LLVM names its nodes after the address of the object they came from, so the + same function printed twice gives two different files. Everything downstream + of this function uses `n0`, `n1` and so on, in the order the nodes appear, + which is stable enough to put in a test. + """ + caption = "" + nodes: list[Node] = [] + edges: list[Edge] = [] + keys: dict[str, str] = {} + + def key_for(pointer: str) -> str: + if pointer not in keys: + keys[pointer] = f"n{len(keys)}" + return keys[pointer] + + for line in dot.splitlines(): + if not caption: + heading = GRAPH_LABEL.match(line) + if heading: + caption = _unescape(heading.group(1)).strip() + continue + edge = EDGE.match(line) + if edge: + label = edge.group("port") or "" + attrs = edge.group("attrs") or "" + named = LABEL.search(attrs) + if named: + label = _unescape(named.group(1)).strip().strip("[]") + edges.append(Edge(src=key_for(edge.group(1)), dst=key_for(edge.group(3)), label=label)) + continue + found = NODE.match(line) + if found: + nodes.append(_node(key_for(found.group(1)), found.group("attrs"))) + return caption, nodes, edges + + +def _label_ports(nodes: list[Node], edges: list[Edge]) -> None: + """Turn `:s0` and `:s1` into the T and F the record label spelled out.""" + by_key = {node.key: node for node in nodes} + for edge in edges: + node = by_key.get(edge.src) + if node and edge.label in node.ports: + edge.label = node.ports[edge.label] + + +# -- getting one -------------------------------------------------------------- + + +class GraphError(RuntimeError): + """Asked for a graph that this module or this pass did not produce.""" + + +def graph(module: Module | str, kind: str = "cfg", function: str | None = None) -> Graph: + """Run one of LLVM's graph printers and hand back what it drew. + + `function` picks between the files when the module has more than one, by the + name in the file `opt` wrote. Leaving it out is fine for a one function + module and an error otherwise, with the available names in the message, + because guessing which function the reader meant is how a lesson ends up + quietly showing the wrong graph. + """ + if kind not in KINDS: + raise GraphError(f"{kind!r} is not one of {', '.join(KINDS)}.") + pass_name, pattern, _ = KINDS[kind] + text = module.text if isinstance(module, Module) else module + + with tempfile.TemporaryDirectory(prefix="irx-graph-") as tmp: + work = Path(tmp) + (work / "in.ll").write_text(text, encoding="utf-8") + run("opt", f"-passes={pass_name}", "-disable-output", "in.ll", cwd=work) + written = sorted(work.glob(pattern)) + if not written: + raise GraphError( + f"`opt -passes={pass_name}` wrote no graph for this module. " + "The printers only emit for functions with a body, so a module of " + "declarations produces nothing." + ) + chosen = _choose(written, kind, function) + dot = chosen.read_text(encoding="utf-8") + + caption, nodes, edges = parse(dot) + # The call graph puts the input file name in its caption, and the input file + # here is a temporary directory nobody will ever see. Say the module's name + # instead, which is what the reader called it. + if isinstance(module, Module): + caption = caption.replace("in.ll", module.name) + _label_ports(nodes, edges) + _mark_back_edges(nodes, edges) + return Graph(kind=kind, caption=caption, nodes=nodes, edges=edges, dot=dot) + + +def _choose(written: list[Path], kind: str, function: str | None) -> Path: + """One file out of the several a multi function module produces.""" + def named(path: Path) -> str: + # `.f.dot`, `dom.f.dot`, `ddg.f..dot`, `in.ll.callgraph.dot`. The name is + # whatever is left once the prefix and the extensions are taken off. + stem = path.name.removesuffix(".dot") + for prefix in ("dom.", "postdom.", "ddg."): + stem = stem.removeprefix(prefix) + return stem.strip(".").removesuffix(".callgraph") + + if function is not None: + for path in written: + if named(path) == function: + return path + raise GraphError( + f"No {kind} graph for {function!r}. This module has " + f"{', '.join(repr(named(p)) for p in written)}." + ) + if len(written) > 1: + raise GraphError( + f"This module has more than one {kind} graph, so say which: " + f"{', '.join(repr(named(p)) for p in written)}." + ) + return written[0] + + +def cfg(module: Module | str, function: str | None = None, *, bodies: bool = True) -> Graph: + """The control flow graph, which is the one Part I wants.""" + return graph(module, "cfg" if bodies else "cfg-only", function) + + +def dom(module: Module | str, function: str | None = None) -> Graph: + return graph(module, "dom", function) + + +def ddg(module: Module | str, function: str | None = None) -> Graph: + return graph(module, "ddg", function) + + +def callgraph(module: Module | str) -> Graph: + return graph(module, "callgraph") + + +# -- ranking ------------------------------------------------------------------ + + +def _mark_back_edges(nodes: list[Node], edges: list[Edge]) -> None: + """A depth first walk, and every edge back into the current stack is a loop. + + Done here rather than at drawing time because `back` is a fact about the + graph and shows up in the text form too. A back edge in a CFG is a loop, and + a reader who cannot see which edge that is has not been shown the loop. + """ + out: dict[str, list[Edge]] = {node.key: [] for node in nodes} + for edge in edges: + if edge.src in out: + out[edge.src].append(edge) + + colour: dict[str, int] = dict.fromkeys(out, 0) # 0 white, 1 on the stack, 2 done + for start in out: + if colour[start]: + continue + stack: list[tuple[str, int]] = [(start, 0)] + colour[start] = 1 + while stack: + key, index = stack[-1] + if index == len(out[key]): + colour[key] = 2 + stack.pop() + continue + stack[-1] = (key, index + 1) + edge = out[key][index] + if colour.get(edge.dst, 2) == 1: + edge.back = True + elif colour.get(edge.dst, 2) == 0: + colour[edge.dst] = 1 + stack.append((edge.dst, 0)) + + +def _ranks(nodes: list[Node], edges: list[Edge]) -> dict[str, int]: + """How far down the page each node goes: one past its furthest predecessor. + + Longest path rather than shortest, so an edge never points upward except the + back edges, which were taken out of the graph before this ran. + """ + forward = [e for e in edges if not e.back and e.src != e.dst] + incoming: dict[str, int] = dict.fromkeys((n.key for n in nodes), 0) + for edge in forward: + if edge.dst in incoming: + incoming[edge.dst] += 1 + + rank = dict.fromkeys(incoming, 0) + ready = [n.key for n in nodes if incoming[n.key] == 0] + seen = 0 + while ready: + key = ready.pop(0) + seen += 1 + for edge in forward: + if edge.src != key or edge.dst not in incoming: + continue + rank[edge.dst] = max(rank[edge.dst], rank[key] + 1) + incoming[edge.dst] -= 1 + if incoming[edge.dst] == 0: + ready.append(edge.dst) + # A cycle the back edge walk did not break would leave nodes unplaced. It + # should not happen, and if it does, putting them at the bottom is better + # than dropping them off the picture. + if seen < len(nodes): + floor = max(rank.values(), default=0) + 1 + for node in nodes: + if incoming[node.key] > 0: + rank[node.key] = floor + return rank + + +# -- drawing ------------------------------------------------------------------ + +CHAR_W = 7.0 # ui-monospace at 11.5px, near enough for a box width +LINE_H = 15.0 +TITLE_H = 19.0 +PAD_X, PAD_Y = 9.0, 7.0 +GAP_X, GAP_Y = 34.0, 44.0 +MARGIN = 12.0 +FRAME = "#d0d7de" +INK = "#24292f" +QUIET = "#57606a" +FLOW = "#0969da" +LOOP = "#8250df" + + +@dataclass +class Box: + node: Node + x: float = 0.0 + y: float = 0.0 + w: float = 0.0 + h: float = 0.0 + + @property + def cx(self) -> float: + return self.x + self.w / 2 + + @property + def cy(self) -> float: + return self.y + self.h / 2 + + +def _boxes(nodes: list[Node], rank: dict[str, int]) -> list[Box]: + boxes = [] + for node in nodes: + width = max([len(line) for line in node.lines] + [len(node.title) + 2]) + boxes.append( + Box( + node=node, + w=width * CHAR_W + 2 * PAD_X, + h=(TITLE_H if node.title else 0) + len(node.lines) * LINE_H + 2 * PAD_Y, + ) + ) + + rows: dict[int, list[Box]] = {} + for box in boxes: + rows.setdefault(rank[box.node.key], []).append(box) + + top = MARGIN + for index in sorted(rows): + row = rows[index] + span = sum(b.w for b in row) + GAP_X * (len(row) - 1) + left = -span / 2 + for box in row: + box.x, box.y = left, top + left += box.w + GAP_X + top += max(b.h for b in row) + GAP_Y + + shift = MARGIN - min(b.x for b in boxes) + for box in boxes: + box.x += shift + return boxes + + +def _tspans(line: str, x: float, y: float) -> str: + """One line of IR as coloured SVG, using the same palette as the diff view.""" + pieces = [] + position = 0 + for match in TOKEN.finditer(line): + pieces.append((line[position : match.start()], "")) + kind = match.lastgroup or "" + value = match.group() + if kind == "word": + kind = "keyword" if value in KEYWORDS else "" + pieces.append((value, COLOUR.get(kind, ""))) + position = match.end() + pieces.append((line[position:], "")) + + spans = [] + for text, colour in pieces: + if not text: + continue + fill = f' fill="{colour}"' if colour else "" + weight = ' font-weight="600"' if colour == COLOUR["keyword"] else "" + spans.append(f'{html.escape(text)}') + return f'{"".join(spans)}' + + +def _curve(a: tuple[float, float], b: tuple[float, float]) -> str: + lift = min(28.0, max(10.0, (b[1] - a[1]) / 2)) + return ( + f"M {a[0]:.1f} {a[1]:.1f} C {a[0]:.1f} {a[1] + lift:.1f}, " + f"{b[0]:.1f} {b[1] - lift:.1f}, {b[0]:.1f} {b[1]:.1f}" + ) + + +def _side(box_a: Box, box_b: Box, lane: float, side: int) -> str: + """Out of one side, along a lane in the margin, and back in at the same side. + + For the two edges that cannot be a short hop between neighbouring rows: a + back edge, which goes up, and a forward edge that skips a row, which would + otherwise be drawn straight through whatever box is sitting in between. + """ + edge_a = box_a.x + box_a.w if side > 0 else box_a.x + edge_b = box_b.x + box_b.w if side > 0 else box_b.x + corner = 8 * side + up = -8 if box_b.cy < box_a.cy else 8 + return ( + f"M {edge_a:.1f} {box_a.cy:.1f} H {lane - corner:.1f} " + f"Q {lane:.1f} {box_a.cy:.1f} {lane:.1f} {box_a.cy + up:.1f} " + f"V {box_b.cy - up:.1f} Q {lane:.1f} {box_b.cy:.1f} {lane - corner:.1f} {box_b.cy:.1f} " + f"H {edge_b:.1f}" + ) + + +def _self(box: Box, lane: float) -> str: + """A block that branches to itself, which is the tightest loop there is.""" + top, bottom = box.y + box.h / 3, box.y + 2 * box.h / 3 + right = box.x + box.w + return ( + f"M {right:.1f} {bottom:.1f} H {lane - 8:.1f} Q {lane:.1f} {bottom:.1f} " + f"{lane:.1f} {bottom - 8:.1f} V {top + 8:.1f} Q {lane:.1f} {top:.1f} " + f"{lane - 8:.1f} {top:.1f} H {right:.1f}" + ) + + +# How far apart two edges leaving the same box are pulled, so that a T and an F +# out of one terminator are two arrows rather than one arrow with two labels +# printed on top of each other. +SPREAD = 26.0 +LANE = 15.0 + + +def _spread(box: Box, count: int) -> float: + """How far apart to put them, without walking off the edge of the box.""" + if count < 2: + return 0.0 + return min(SPREAD, max(10.0, (box.w - 20) / (count - 1))) + + +def _fan(edges: list[Edge], pick: str) -> dict[int, tuple[int, int]]: + """Which of its siblings each edge is, at whichever end `pick` names.""" + groups: dict[str, list[Edge]] = {} + for edge in edges: + groups.setdefault(getattr(edge, pick), []).append(edge) + spot = {} + for group in groups.values(): + for index, edge in enumerate(group): + spot[id(edge)] = (index, len(group)) + return spot + + +def _svg(g: Graph) -> str: + rank = _ranks(g.nodes, g.edges) + boxes = _boxes(g.nodes, rank) + at = {box.node.key: box for box in boxes} + + loops, skips, short = [], [], [] + for edge in g.edges: + if edge.back: + loops.append(edge) + elif rank[edge.dst] - rank[edge.src] > 1: + skips.append(edge) + else: + short.append(edge) + # The fan is over the short edges only. The long ones leave from the side of + # the box rather than the bottom, so counting them here would push the short + # ones off centre to make room for an arrow that is not there. + from_source = _fan(short, "src") + to_target = _fan(short, "dst") + + # Lanes live outside the boxes, so the drawing has to make room for them + # before anything is placed. Skipping edges go down the left, loops up the + # right, which keeps the two kinds of long edge from sharing a corridor. + left_pad = LANE * (len(skips) + 1) if skips else 0.0 + for box in boxes: + box.x += left_pad + right = max(box.x + box.w for box in boxes) + # The extra 24 is where the T or the F on a back edge goes. Without it the + # label sits on top of the dashed line it belongs to. + width = right + MARGIN + (LANE * len(loops) + 24 if loops else 0.0) + height = max(box.y + box.h for box in boxes) + MARGIN + + # A content hash, not id() or hash(), so the same graph gives the same + # marker id every time. A notebook that is rebuilt should have a byte + # identical diff or none at all. + ident = "irxg" + hashlib.sha256(g.dot.encode()).hexdigest()[:8] + parts = [ + f'', + f'' + f'' + "", + ] + + def label_at( + edge: Edge, x: float, y: float, colour: str = FLOW, anchor: str = "middle" + ) -> None: + if edge.label: + parts.append( + f'{html.escape(edge.label)}' + ) + + for index, edge in enumerate(loops): + lane = right + LANE * (index + 1) + source, target = at[edge.src], at[edge.dst] + path = _self(source, lane) if edge.src == edge.dst else _side(source, target, lane, 1) + parts.append( + f'' + ) + middle = source.cy if edge.src == edge.dst else (source.cy + target.cy) / 2 + label_at(edge, lane + 5, middle + 3, LOOP, "start") + + for index, edge in enumerate(skips): + lane = left_pad - LANE * (index + 1) + source, target = at[edge.src], at[edge.dst] + parts.append( + f'' + ) + label_at(edge, lane + 5, (source.cy + target.cy) / 2 + 3, FLOW, "start") + + for edge in short: + a, b = at[edge.src], at[edge.dst] + out_of, out_count = from_source[id(edge)] + into, in_count = to_target[id(edge)] + start = (a.cx + (out_of - (out_count - 1) / 2) * _spread(a, out_count), a.y + a.h) + end = (b.cx + (into - (in_count - 1) / 2) * _spread(b, in_count), b.y - 3) + parts.append( + f'' + ) + # Two rows of labels, alternating. A block with four successors has its + # arrows a couple of dozen pixels apart and the word `def-use` is wider + # than that, so on one row they would sit on top of each other. + label_at(edge, start[0], start[1] + 13 + (out_of % 2) * 12) + + for box in boxes: + node = box.node + parts.append( + f'' + ) + top = box.y + PAD_Y + if node.title: + parts.append( + f'{html.escape(node.title)}' + ) + top += TITLE_H + for offset, line in enumerate(node.lines): + parts.append(_tspans(line, box.x + PAD_X, top + 11 + offset * LINE_H)) + + parts.append("") + return "".join(parts) + + +@dataclass(repr=False) +class Graph: + """What one of LLVM's graph printers produced, as text or as a picture.""" + + kind: str + caption: str + nodes: list[Node] + edges: list[Edge] + dot: str + + @property + def loops(self) -> list[Edge]: + """The back edges. In a CFG, one of these is one loop.""" + return [edge for edge in self.edges if edge.back] + + def successors(self, name: str) -> list[str]: + by_key = {node.key: node for node in self.nodes} + return [ + by_key[edge.dst].name + for edge in self.edges + if by_key.get(edge.src) is not None and by_key[edge.src].name == name + ] + + def __str__(self) -> str: + lines = [self.caption or f"{self.kind} graph", ""] + by_key = {node.key: node for node in self.nodes} + for node in self.nodes: + lines.append(f" {node.name}") + lines += [f" {line}" for line in node.lines] + for edge in self.edges: + if edge.src != node.key: + continue + arrow = "=>" if edge.back else "->" + tag = f"{edge.label} " if edge.label else "" + lines.append(f" {tag}{arrow} {by_key[edge.dst].name}") + lines.append("") + if self.loops: + lines.append(f" {len(self.loops)} back edge(s), drawn with =>") + return "\n".join(lines).rstrip() + "\n" + + def __repr__(self) -> str: + return str(self) + + def svg(self) -> str: + """The picture on its own, for writing to a file.""" + return _svg(self) + + def _repr_html_(self) -> str: + caption = html.escape(self.caption or f"{self.kind} graph") + return ( + f'

{caption}, ' + f"as opt -passes={KINDS[self.kind][0]} drew it

" + f'
{_svg(self)}
' + ) + + +__all__ = ["Edge", "Graph", "GraphError", "Node", "callgraph", "cfg", "ddg", "dom", "graph"] diff --git a/toolkit/irx/proc.py b/toolkit/irx/proc.py index 47de6e4..3aa6283 100644 --- a/toolkit/irx/proc.py +++ b/toolkit/irx/proc.py @@ -117,8 +117,15 @@ def run( stdin: str | None = None, check: bool = True, timeout: int = DEFAULT_TIMEOUT, + cwd: str | Path | None = None, ) -> Result: - """Run one LLVM tool. `tool` is a bare name like `opt`, resolved against the pin.""" + """Run one LLVM tool. `tool` is a bare name like `opt`, resolved against the pin. + + `cwd` is for the handful of things LLVM writes to the working directory + rather than to stdout. The graph printers are the ones this repository + needs: `opt -passes=dot-cfg` writes `.f.dot` next to wherever you are + standing, and there is no flag that redirects it. + """ argv = [str(toolchain.path_to(tool)), *(str(a) for a in args)] start = time.monotonic() try: @@ -128,6 +135,7 @@ def run( capture_output=True, text=True, timeout=timeout, + cwd=None if cwd is None else str(cwd), ) except subprocess.TimeoutExpired: elapsed = time.monotonic() - start diff --git a/toolkit/tests/test_graphs.py b/toolkit/tests/test_graphs.py new file mode 100644 index 0000000..870924f --- /dev/null +++ b/toolkit/tests/test_graphs.py @@ -0,0 +1,219 @@ +"""Tests for irx.graphs. + +Split the same way as the rest of the toolkit tests. The parser and the layout +run against a fixture and need nothing installed, because they are text handling +and should be covered on a machine with no compiler. The rest run `opt` and skip +with a reason when there is no pinned toolchain. + +The fixture below is verbatim `opt -passes=dot-cfg` output, addresses and all. +It has been left exactly as LLVM wrote it, because the point of these tests is +that this module reads what LLVM actually produces rather than what would have +been convenient. +""" + +from __future__ import annotations + +import unittest +import xml.etree.ElementTree as ET + +import irx +from irx import graphs, toolchain + +DOT = r"""digraph "CFG for 'f' function" { + label="CFG for 'f' function"; + + Node0x600002f880a0 [shape=record,color="#3d50c3ff", style=filled, fillcolor="#d24b4070", fontname="Courier",label="{1:\l| %2 = icmp sgt i32 %0, 0\l br i1 %2, label %5, label %3\l|{T|F}}"]; + Node0x600002f880a0:s0 -> Node0x600002f88140; + Node0x600002f880a0:s1 -> Node0x600002f88190; + Node0x600002f88190 [shape=record,label="{3:\l| %4 = phi i32 [ 0, %1 ], [ %12, %5 ]\l ret i32 %4\l}"]; + Node0x600002f88140 [shape=record,label="{5:\l| %6 = phi i32 [ %13, %5 ], [ 0, %1 ]\l br i1 %14, label %3, label %5\l|{T|F}}"]; + Node0x600002f88140:s0 -> Node0x600002f88190; + Node0x600002f88140:s1 -> Node0x600002f88140; +} +""" + + +def toolchain_available() -> bool: + try: + toolchain.bootstrap(verbose=False) + except toolchain.ToolchainError: + return False + return True + + +needs_llvm = unittest.skipUnless( + toolchain_available(), f"no LLVM {toolchain.MAJOR}, set IRX_LLVM_BIN to run these" +) + + +class TestLabelEscapes(unittest.TestCase): + def test_both_kinds_of_line_break_are_a_line_break(self): + self.assertEqual(graphs._unescape(r"a\lb\nc"), "a\nb\nc") + + def test_an_escaped_angle_bracket_is_an_angle_bracket(self): + # The DDG printer writes this way, and a reader who sees + # \ in a picture has been shown the escaping, not the graph. + self.assertEqual(graphs._unescape(r"\"), "") + + def test_a_bar_inside_braces_is_not_a_field(self): + self.assertEqual(graphs._split("a|{b|c}|d"), ["a", "{b|c}", "d"]) + + def test_an_escaped_bar_is_not_a_field_either(self): + # `or` disassembles as a bar in some dumps, and splitting on it would + # cut an instruction in half. + self.assertEqual(graphs._split(r"a\|b|c"), [r"a\|b", "c"]) + + +class TestParse(unittest.TestCase): + def setUp(self): + self.caption, self.nodes, self.edges = graphs.parse(DOT) + graphs._label_ports(self.nodes, self.edges) + graphs._mark_back_edges(self.nodes, self.edges) + + def test_the_caption_is_the_graph_label(self): + self.assertEqual(self.caption, "CFG for 'f' function") + + def test_every_node_arrives_with_its_block_number_and_its_instructions(self): + self.assertEqual([n.title for n in self.nodes], ["1", "3", "5"]) + self.assertEqual(self.nodes[1].lines[-1], " ret i32 %4") + + def test_no_pointer_survives_the_parse(self): + # LLVM names nodes after addresses, so the same function printed twice + # gives two different files. Nothing downstream may depend on that. + keys = {n.key for n in self.nodes} + self.assertEqual(keys, {"n0", "n1", "n2"}) + self.assertNotIn("Node0x", str(self.edges)) + + def test_the_ports_become_the_true_and_false_arms(self): + labels = sorted(e.label for e in self.edges if e.src == "n0") + self.assertEqual(labels, ["F", "T"]) + + def test_the_edge_that_goes_back_into_the_loop_is_marked(self): + back = [(e.src, e.dst) for e in self.edges if e.back] + self.assertEqual(back, [("n1", "n1")]) + + def test_the_text_form_names_blocks_and_draws_back_edges_differently(self): + text = str(graphs.Graph("cfg", self.caption, self.nodes, self.edges, DOT)) + self.assertIn("F => 5", text) + self.assertIn("T -> 3", text) + self.assertIn("1 back edge(s)", text) + + +class TestRanks(unittest.TestCase): + def test_a_node_sits_one_below_its_furthest_predecessor(self): + # Longest path, not shortest. n2 has an edge straight from n0 as well as + # one through n1, and putting it on row one would draw an arrow upward. + nodes = [graphs.Node(key=f"n{i}", title=str(i)) for i in range(3)] + edges = [graphs.Edge("n0", "n1"), graphs.Edge("n1", "n2"), graphs.Edge("n0", "n2")] + self.assertEqual(graphs._ranks(nodes, edges), {"n0": 0, "n1": 1, "n2": 2}) + + def test_a_back_edge_does_not_count_towards_the_ranking(self): + nodes = [graphs.Node(key=f"n{i}", title=str(i)) for i in range(2)] + edges = [graphs.Edge("n0", "n1"), graphs.Edge("n1", "n0", back=True)] + self.assertEqual(graphs._ranks(nodes, edges), {"n0": 0, "n1": 1}) + + +class TestSvg(unittest.TestCase): + def setUp(self): + caption, nodes, edges = graphs.parse(DOT) + graphs._label_ports(nodes, edges) + graphs._mark_back_edges(nodes, edges) + self.graph = graphs.Graph("cfg", caption, nodes, edges, DOT) + self.svg = self.graph.svg() + + def test_it_is_well_formed_xml(self): + root = ET.fromstring(self.svg) + self.assertTrue(root.tag.endswith("svg")) + + def test_there_is_a_box_for_every_block_and_a_path_for_every_edge(self): + root = ET.fromstring(self.svg) + rects = root.findall(".//{http://www.w3.org/2000/svg}rect") + paths = root.findall(".//{http://www.w3.org/2000/svg}path") + self.assertEqual(len(rects), 3) + # Four edges, plus the two arrowhead markers in the defs block. + self.assertEqual(len(paths), 6) + + def test_nothing_in_it_is_a_script(self): + # This ends up inside a saved notebook and inside static HTML on the + # site. Colab strips script, and the site should not be serving any. + self.assertNotIn("