From b69eb834fb13338cb6fce192af458be3c089e218 Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 18:19:08 +0200 Subject: [PATCH 1/8] fix metric plot super class --- ablate/blocks/figure_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ablate/blocks/figure_blocks.py b/ablate/blocks/figure_blocks.py index d9a806a..c109c2f 100644 --- a/ablate/blocks/figure_blocks.py +++ b/ablate/blocks/figure_blocks.py @@ -19,7 +19,7 @@ class AbstractFigureBlock(AbstractBlock, ABC): def build(self, runs: List[Run]) -> pd.DataFrame: ... -class MetricPlot(AbstractBlock): +class MetricPlot(AbstractFigureBlock): def __init__( self, metric: AbstractMetric | List[AbstractMetric], From 577461130593ac4dc2c4c6c0031cabb92810c72a Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:03:34 +0200 Subject: [PATCH 2/8] add private heading block --- ablate/blocks/text_blocks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ablate/blocks/text_blocks.py b/ablate/blocks/text_blocks.py index 6e89282..764c085 100644 --- a/ablate/blocks/text_blocks.py +++ b/ablate/blocks/text_blocks.py @@ -22,22 +22,25 @@ def build(self, runs: List[Run]) -> str: return self.text.strip() -class H1(AbstractTextBlock): ... +class _Heading(AbstractTextBlock): ... -class H2(AbstractTextBlock): ... +class H1(_Heading): ... -class H3(AbstractTextBlock): ... +class H2(_Heading): ... -class H4(AbstractTextBlock): ... +class H3(_Heading): ... -class H5(AbstractTextBlock): ... +class H4(_Heading): ... -class H6(AbstractTextBlock): ... +class H5(_Heading): ... + + +class H6(_Heading): ... class Text(AbstractTextBlock): ... From 2849293b3b537fd2f676748f469acd6cea1a43a2 Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:04:31 +0200 Subject: [PATCH 3/8] add report --- ablate/report.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ablate/report.py diff --git a/ablate/report.py b/ablate/report.py new file mode 100644 index 0000000..6f8fc7d --- /dev/null +++ b/ablate/report.py @@ -0,0 +1,36 @@ +from typing import List + +from typing_extensions import Self + +from ablate.blocks import AbstractBlock +from ablate.core.types import Run + + +class Report: + def __init__(self, runs: List[Run]) -> None: + """Report mapping a list of runs to a list of blocks. + + Args: + runs: List of runs to be associated with the report. + """ + self.runs = runs + self.blocks: List[AbstractBlock] = [] + + def add(self, *blocks: AbstractBlock) -> Self: + """Add one or more blocks to the report. + + Returns: + The updated report with the added blocks. + """ + for block in blocks: + self.blocks.append(block) + return self + + def __iadd__(self, block: AbstractBlock) -> Self: + self.blocks.append(block) + return self + + def __add__(self, block: AbstractBlock) -> Self: + r = Report(self.runs) + r.blocks = self.blocks + [block] + return r From 83b27f03fe2bc2f1e6ffd51a9252e2c112dec361 Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:04:39 +0200 Subject: [PATCH 4/8] add report tests --- tests/test_report.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/test_report.py diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..c353439 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,34 @@ +from typing import List + +from ablate.blocks import Text +from ablate.core.types import Run +from ablate.report import Report + + +def make_runs() -> List[Run]: + return [ + Run(id="a", params={"model": "resnet", "seed": 1}, metrics={"accuracy": 0.7}), + Run(id="b", params={"model": "resnet", "seed": 2}, metrics={"accuracy": 0.8}), + ] + + +def test_report_add() -> None: + report = Report(make_runs()) + report.add(Text("Test 1"), Text("Test 2")) + assert len(report.blocks) == 2 + assert isinstance(report.blocks[0], Text) + + +def test_report_iadd() -> None: + report = Report(make_runs()) + report += Text("Test") + assert len(report.blocks) == 1 + assert isinstance(report.blocks[0], Text) + + +def test_report_add_operator() -> None: + report = Report(make_runs()) + new_report = report + Text("Test") + assert len(report.blocks) == 0 + assert len(new_report.blocks) == 1 + assert isinstance(new_report.blocks[0], Text) From 9df78eec04c903e566ae5a1e0d99d03b499214bb Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:05:01 +0200 Subject: [PATCH 5/8] add abstract exporter --- ablate/exporters/abstract_exporter.py | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 ablate/exporters/abstract_exporter.py diff --git a/ablate/exporters/abstract_exporter.py b/ablate/exporters/abstract_exporter.py new file mode 100644 index 0000000..112aaee --- /dev/null +++ b/ablate/exporters/abstract_exporter.py @@ -0,0 +1,96 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable, List + +from ablate.blocks import ( + AbstractBlock, + AbstractFigureBlock, + AbstractTableBlock, + AbstractTextBlock, +) +from ablate.core.types import Run +from ablate.report import Report + + +class AbstractExporter(ABC): + @abstractmethod + def export(self, report: Report) -> None: + """Export the report. + + Should call the `render_blocks` to generate the content of the report. + + Args: + report: The report to be exported. + """ + + def render_blocks(self, report: Report) -> List[Any]: + """Render a blocks of the report. + + Args: + report: The report to be rendered. + + Raises: + ValueError: If the block type is not supported. + + Returns: + List of rendered blocks. + """ + render_map = { + AbstractTextBlock: self.render_text, + AbstractTableBlock: self.render_table, + AbstractFigureBlock: self.render_figure, + } + content = [] + for block in report.blocks: + for block_type, render_fn in render_map.items(): + if isinstance(block, block_type): + content.append(self._apply_render_fn(block, render_fn, report.runs)) + break + else: + raise ValueError(f"Unknown block type: '{type(block)}'.") + return content + + @staticmethod + def _apply_render_fn( + block: AbstractBlock, + fn: Callable[[AbstractBlock], Any], + runs: List[Run], + ) -> Any: + if block.runs: + return fn(block, block.runs) + return fn(block, runs) + + @abstractmethod + def render_text(self, block: AbstractTextBlock, runs: List[Run]) -> Any: + """Render a text block. + + Args: + block: The text block to be rendered. + runs: The list of runs to be used for the block. + + Returns: + The rendered text block. + """ + + @abstractmethod + def render_table(self, block: AbstractTableBlock, runs: List[Run]) -> Any: + """Render a table block. + + Args: + block: The table block to be rendered. + runs: The list of runs to be used for the block. + + Returns: + The rendered table block. + """ + + @abstractmethod + def render_figure(self, block: AbstractFigureBlock, runs: List[Run]) -> Any: + """Render a figure block. + + Args: + block: The figure block to be rendered. + runs: The list of runs to be used for the block. + + Returns: + The rendered figure block. + """ From fcd432d338fc798289f3cb1fb7b5c50903c2f838 Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:05:13 +0200 Subject: [PATCH 6/8] add markdown exporter --- ablate/__init__.py | 5 +- ablate/blocks/abstract_block.py | 3 +- ablate/exporters/__init__.py | 5 ++ ablate/exporters/markdown_exporter.py | 69 +++++++++++++++++++++++++++ ablate/exporters/utils.py | 58 ++++++++++++++++++++++ 5 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 ablate/exporters/__init__.py create mode 100644 ablate/exporters/markdown_exporter.py create mode 100644 ablate/exporters/utils.py diff --git a/ablate/__init__.py b/ablate/__init__.py index 9763d79..447c48e 100644 --- a/ablate/__init__.py +++ b/ablate/__init__.py @@ -1,6 +1,7 @@ -from . import blocks, queries, sources +from . import blocks, exporters, queries, sources +from .report import Report -__all__ = ["blocks", "queries", "sources"] +__all__ = ["blocks", "exporters", "queries", "Report", "sources"] __version__ = "0.1.0" diff --git a/ablate/blocks/abstract_block.py b/ablate/blocks/abstract_block.py index b035c6b..aa0e536 100644 --- a/ablate/blocks/abstract_block.py +++ b/ablate/blocks/abstract_block.py @@ -16,7 +16,8 @@ def __init__(self, runs: List[Run] | None = None) -> None: @abstractmethod def build(self, runs: List[Run]) -> Any: - """Build the intermediate representation of the block, ready for rendering. + """Build the intermediate representation of the block, ready for rendering + and export. Args: runs: List of runs to be used for the block. diff --git a/ablate/exporters/__init__.py b/ablate/exporters/__init__.py new file mode 100644 index 0000000..78ab4e8 --- /dev/null +++ b/ablate/exporters/__init__.py @@ -0,0 +1,5 @@ +from .abstract_exporter import AbstractExporter +from .markdown_exporter import Markdown + + +__all__ = ["AbstractExporter", "Markdown"] diff --git a/ablate/exporters/markdown_exporter.py b/ablate/exporters/markdown_exporter.py new file mode 100644 index 0000000..1cbf12a --- /dev/null +++ b/ablate/exporters/markdown_exporter.py @@ -0,0 +1,69 @@ +from pathlib import Path +from typing import List + +from ablate.blocks import ( + AbstractFigureBlock, + AbstractTableBlock, + AbstractTextBlock, + MetricPlot, + Text, +) +from ablate.blocks.text_blocks import _Heading +from ablate.core.types import Run +from ablate.exporters.abstract_exporter import AbstractExporter +from ablate.report import Report + +from .utils import HEADING_LEVELS, render_metric_plot + + +class Markdown(AbstractExporter): + def __init__( + self, + output_path: str = "report.md", + assets_dir: str | None = None, + ) -> None: + """Export the report as a markdown file. + + Args: + output_path: The path to the output markdown file. Defaults to "report.md". + assets_dir: The directory to store the assets (figures, etc.). If None, + defaults to the parent directory of the output file with a ".ablate" + subdirectory. Defaults to None. + """ + self.output_path = Path(output_path) + self.assets_dir = ( + Path(assets_dir) if assets_dir else self.output_path.parent / ".ablate" + ) + self.assets_dir.mkdir(exist_ok=True) + + def export(self, report: Report) -> None: + content = self.render_blocks(report) + with self.output_path.open("w", encoding="utf-8") as f: + for block_output in content: + f.write(block_output) + f.write("\n\n") + + def render_text(self, block: AbstractTextBlock, runs: List[Run]) -> str: + if isinstance(block, Text): + return block.build(runs) + if isinstance(block, _Heading): + return f"{'#' * HEADING_LEVELS[type(block)]} {block.build(runs)}" + raise NotImplementedError(f"Unsupported text block: '{type(block)}'.") + + def render_table(self, block: AbstractTableBlock, runs: List[Run]) -> str: + return block.build(runs).to_markdown(index=False) + + def render_figure(self, block: AbstractFigureBlock, runs: List[Run]) -> str: + if not isinstance(block, MetricPlot): + raise NotImplementedError(f"Unsupported figure block: '{type(block)}'.") + + filename = render_metric_plot( + block.build(runs), + self.assets_dir, + type(block).__name__, + ) + if filename is None: + return ( + f"*No data available for {', '.join(m.label for m in block.metrics)}*" + ) + return f"![{filename}](.ablate/{filename})" diff --git a/ablate/exporters/utils.py b/ablate/exporters/utils.py new file mode 100644 index 0000000..11fe836 --- /dev/null +++ b/ablate/exporters/utils.py @@ -0,0 +1,58 @@ +import hashlib +from pathlib import Path + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +from ablate.blocks import H1, H2, H3, H4, H5, H6 + + +HEADING_LEVELS = {H1: 1, H2: 2, H3: 3, H4: 4, H5: 5, H6: 6} + +DEFAULT_PLOT_STYLE = { + "style": "whitegrid", + "context": "paper", + "palette": "muted", + "dpi": 300, + "font_scale": 0.8, +} + + +def apply_default_plot_style() -> None: + sns.set_style(DEFAULT_PLOT_STYLE["style"]) + sns.set_context( + DEFAULT_PLOT_STYLE["context"], font_scale=DEFAULT_PLOT_STYLE["font_scale"] + ) + sns.set_palette(DEFAULT_PLOT_STYLE["palette"]) + plt.rcParams["figure.dpi"] = DEFAULT_PLOT_STYLE["dpi"] + + +def render_metric_plot( + df: pd.DataFrame, + output_dir: Path, + name_prefix: str, +) -> str | None: + apply_default_plot_style() + if df.empty: + return None + + fig, ax = plt.subplots() + sns.lineplot( + data=df, + x="step", + y="value", + hue="run", + style="metric" if df["metric"].nunique() > 1 else None, + ax=ax, + ) + ax.set_xlabel("Step") + ax.set_ylabel("Value") + ax.legend(title="Run", loc="best", frameon=False) + plt.tight_layout() + + h = hashlib.md5(df.to_csv(index=False).encode("utf-8")).hexdigest()[:12] + filename = f"{name_prefix}_{h}.png" + fig.savefig(output_dir / filename) + plt.close(fig) + return filename From 4f97b75ac644282409b18a5cd59a15fb3f555e0e Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:05:23 +0200 Subject: [PATCH 7/8] add markdown exporter tests --- tests/exporters/test_markdown_exporter.py | 152 ++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/exporters/test_markdown_exporter.py diff --git a/tests/exporters/test_markdown_exporter.py b/tests/exporters/test_markdown_exporter.py new file mode 100644 index 0000000..6a2d57a --- /dev/null +++ b/tests/exporters/test_markdown_exporter.py @@ -0,0 +1,152 @@ +from pathlib import Path +import re +from typing import List + +import matplotlib.pyplot as plt +import pytest + +from ablate.blocks import ( + H1, + H2, + AbstractFigureBlock, + AbstractTextBlock, + MetricPlot, + Table, + Text, +) +from ablate.core.types import Run +from ablate.exporters import Markdown +from ablate.queries import Metric, Param +from ablate.report import Report + + +class DummyBlock: + pass + + +class DummyTextBlock(AbstractTextBlock): + def build(self, runs: List[Run]) -> str: + return "not supported" + + +class DummyFigureBlock(AbstractFigureBlock): + def build(self, runs: List[Run]) -> str: + return "not a dataframe" + + +@pytest.fixture +def runs() -> list[Run]: + return [ + Run( + id="run1", + params={"model": "resnet"}, + metrics={"accuracy": 0.8}, + temporal={"accuracy": [(0, 0.5), (1, 0.8)]}, + ), + Run( + id="run2", + params={"model": "resnet"}, + metrics={"accuracy": 0.9}, + temporal={"accuracy": [(0, 0.6), (1, 0.9)]}, + ), + ] + + +def test_export_text_blocks(tmp_path: Path, runs: List[Run]) -> None: + report = Report(runs).add(H1("Heading 1"), Text("Some paragraph text.")) + out_path = tmp_path / "report.md" + Markdown(output_path=out_path).export(report) + + content = out_path.read_text() + assert "# Heading 1" in content + assert "Some paragraph text." in content + + +def test_export_table_block(tmp_path: Path, runs: List[Run]) -> None: + table = Table( + columns=[ + Param("model", label="Model"), + Metric("accuracy", direction="max", label="Accuracy"), + ] + ) + report = Report(runs).add(table) + out_path = tmp_path / "report.md" + Markdown(output_path=out_path).export(report) + + content = out_path.read_text().replace("\r\n", "\n") + assert re.search(r"\|\s*Model\s*\|\s*Accuracy\s*\|", content) + pattern = ( + r"\|\s*resnet\s*\|\s*0\.8\s*\|\s*\n" + r"\|\s*resnet\s*\|\s*0\.9\s*\|" + ) + assert re.search(pattern, content) + + +def test_export_figure_block(tmp_path: Path, runs: List[Run]) -> None: + plot = MetricPlot(Metric("accuracy", direction="max"), identifier=Param("model")) + report = Report(runs).add(plot) + out_path = tmp_path / "report.md" + exporter = Markdown(output_path=out_path) + exporter.export(report) + + content = out_path.read_text() + assert "![MetricPlot_" in content + asset_path = tmp_path / ".ablate" + assert asset_path.exists() + files = list(asset_path.glob("MetricPlot_*.png")) + assert len(files) == 1 + img = plt.imread(files[0]) + assert img.shape[-1] in {3, 4} + + +def test_export_figure_block_empty(tmp_path: Path) -> None: + empty_run = Run(id="x", params={}, metrics={}, temporal={}) + plot = MetricPlot(Metric("accuracy", direction="max"), identifier=Param("model")) + report = Report([empty_run]).add(plot) + out_path = tmp_path / "report.md" + Markdown(output_path=out_path).export(report) + + content = out_path.read_text() + assert "*No data available for accuracy*" in content + + +def test_unknown_block_raises(tmp_path: Path, runs: List[Run]) -> None: + report = Report(runs) + report += DummyBlock() + with pytest.raises(ValueError, match="Unknown block type"): + Markdown(output_path=tmp_path / "out.md").export(report) + + +def test_unsupported_figure_block_raises(tmp_path: Path, runs: List[Run]) -> None: + report = Report(runs).add(DummyFigureBlock()) + exporter = Markdown(output_path=tmp_path / "out.md") + with pytest.raises(NotImplementedError, match="Unsupported figure block"): + exporter.export(report) + + +def test_unsupported_text_block_raises(tmp_path: Path, runs: List[Run]) -> None: + report = Report(runs).add(DummyTextBlock("oops")) + exporter = Markdown(output_path=tmp_path / "out.md") + with pytest.raises(NotImplementedError, match="Unsupported text block"): + exporter.export(report) + + +def test_block_level_runs_override_global(tmp_path: Path, runs: List[Run]) -> None: + scoped_runs = [runs[0]] + report = Report(runs).add( + Text("global"), + Table([Param("model"), Metric("accuracy", "max")], runs=scoped_runs), + ) + out_path = tmp_path / "report.md" + Markdown(output_path=out_path).export(report) + content = out_path.read_text() + assert "resnet" in content + assert content.count("resnet") == 1 + + +def test_export_heading_variants(tmp_path: Path, runs: List[Run]) -> None: + report = Report(runs).add(H2("Section Title")) + out_path = tmp_path / "headings.md" + Markdown(output_path=out_path).export(report) + content = out_path.read_text() + assert "## Section Title" in content From d2f7e67fcff62499fd5fa90dccfdaea6c1837da0 Mon Sep 17 00:00:00 2001 From: Simon Rampp Date: Sun, 11 May 2025 20:24:29 +0200 Subject: [PATCH 8/8] make mypy happy --- ablate/exporters/abstract_exporter.py | 7 ++++--- ablate/exporters/utils.py | 18 ++++-------------- ablate/report.py | 12 ++++++++---- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/ablate/exporters/abstract_exporter.py b/ablate/exporters/abstract_exporter.py index 112aaee..3117c61 100644 --- a/ablate/exporters/abstract_exporter.py +++ b/ablate/exporters/abstract_exporter.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Callable, List +from typing import Any, Callable, List, cast from ablate.blocks import ( AbstractBlock, @@ -43,7 +43,8 @@ def render_blocks(self, report: Report) -> List[Any]: for block in report.blocks: for block_type, render_fn in render_map.items(): if isinstance(block, block_type): - content.append(self._apply_render_fn(block, render_fn, report.runs)) + fn = cast("Callable[[AbstractBlock, List[Run]], Any]", render_fn) + content.append(self._apply_render_fn(block, fn, report.runs)) break else: raise ValueError(f"Unknown block type: '{type(block)}'.") @@ -52,7 +53,7 @@ def render_blocks(self, report: Report) -> List[Any]: @staticmethod def _apply_render_fn( block: AbstractBlock, - fn: Callable[[AbstractBlock], Any], + fn: Callable[[AbstractBlock, List[Run]], Any], runs: List[Run], ) -> Any: if block.runs: diff --git a/ablate/exporters/utils.py b/ablate/exporters/utils.py index 11fe836..e710e68 100644 --- a/ablate/exporters/utils.py +++ b/ablate/exporters/utils.py @@ -10,22 +10,12 @@ HEADING_LEVELS = {H1: 1, H2: 2, H3: 3, H4: 4, H5: 5, H6: 6} -DEFAULT_PLOT_STYLE = { - "style": "whitegrid", - "context": "paper", - "palette": "muted", - "dpi": 300, - "font_scale": 0.8, -} - def apply_default_plot_style() -> None: - sns.set_style(DEFAULT_PLOT_STYLE["style"]) - sns.set_context( - DEFAULT_PLOT_STYLE["context"], font_scale=DEFAULT_PLOT_STYLE["font_scale"] - ) - sns.set_palette(DEFAULT_PLOT_STYLE["palette"]) - plt.rcParams["figure.dpi"] = DEFAULT_PLOT_STYLE["dpi"] + sns.set_style("whitegrid") + sns.set_context("paper", font_scale=0.8) + sns.set_palette("muted") + plt.rcParams["figure.dpi"] = 300 def render_metric_plot( diff --git a/ablate/report.py b/ablate/report.py index 6f8fc7d..2fa8a0a 100644 --- a/ablate/report.py +++ b/ablate/report.py @@ -1,9 +1,13 @@ -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List from typing_extensions import Self -from ablate.blocks import AbstractBlock -from ablate.core.types import Run + +if TYPE_CHECKING: + from ablate.blocks import AbstractBlock + from ablate.core.types import Run class Report: @@ -30,7 +34,7 @@ def __iadd__(self, block: AbstractBlock) -> Self: self.blocks.append(block) return self - def __add__(self, block: AbstractBlock) -> Self: + def __add__(self, block: AbstractBlock) -> Report: r = Report(self.runs) r.blocks = self.blocks + [block] return r