docs: integrate myst-nb notebook rendering infrastructure (#884)

.gitignore

@@ -67,6 +67,7 @@ dmypy.json
 # Sphinx
 docs/_build/
 docs/source/api/generated/
+docs/source/tutorials/
 
 # docs/_doctrees/
 # docs/_static_gen/
@@ -126,8 +127,14 @@ certificates/
 # Benchmark outputs
 benchmark_results*/
 
+# Local dataset files
+liander_dataset/
+
 # Mlflow
 /mlflow
 /mlflow_artifacts_local
 
-.github/instructions
+.github/instructions
+
+# Jupyter notebook cache (myst-nb execution outputs)
+.jupyter_cache/

REUSE.toml

@@ -6,6 +6,7 @@ path = [
     ".python-version",
     "uv.lock",
     "examples/*/*.ipynb",
+    "examples/tutorials/*.py",
 ]
 precedence = "override"
 SPDX-FileCopyrightText = "2025 Contributors to the OpenSTEF project <openstef@lfenergy.org>"

docs/pyproject.toml

@@ -29,7 +29,9 @@ dependencies = [
   "openstef",
   "sphinx-autobuild>=2024.10.3",
   "sphinx-autodoc-typehints>=3.2.0",
-  "myst-parser>=4.0.1",
+  "myst-nb>=1.2.0",
+  "jupyter-cache>=1.0.0",
+  "jupytext>=1.16.0",
 ]
 
 [tool.uv.sources]

docs/source/conf.py

@@ -38,7 +38,7 @@
     "sphinx.ext.mathjax",
     "sphinx.ext.napoleon",
     "sphinx.ext.viewcode",
-    "myst_parser",
+    "myst_nb",
     "sphinx_design",
     "sphinx_copybutton",
     "matplotlib.sphinxext.plot_directive",
@@ -50,7 +50,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
+exclude_patterns = ["conf.py", "**/*.ipynb"]
 
 # Specify how to identify the prompt when copying code snippets
 copybutton_prompt_text = r">>> |\.\.\. "
@@ -124,6 +124,14 @@
     "colon_fence",
 ]
 
+# -- Notebook execution (myst-nb) -------------------------------------------
+nb_custom_formats = {".py": ["jupytext.reads", {"fmt": "py:percent"}]}
+nb_execution_mode = "off"  # TODO(#884): enable "cache" once tutorials are optimized for faster execution
+nb_execution_timeout = 120
+nb_execution_raise_on_error = True
+# TODO(#884): backtesting notebook exceeds timeout — needs rewrite or execution split
+nb_execution_excludepatterns = ["tutorials/backtesting_openstef_with_beam*"]
+
 # Sphinx version switcher
 config = SphinxConfig("../../pyproject.toml", globalns=globals())
 version = config.version
@@ -300,12 +308,17 @@
 # -- Sphinx setup ------------------------------------------------------------
 
 
-def rstjinja(app: Sphinx, _docname: str, source: list[str]) -> None:
+def rstjinja(app: Sphinx, docname: str, source: list[str]) -> None:
     """Render RST files as Jinja templates for variable substitution."""
     # Only process HTML builds
     if app.builder.format != "html":  # type: ignore[attr-defined]
         return
 
+    # Only process .rst sources — skip notebooks/MyST which contain {} literals
+    rst_path = Path(app.srcdir) / f"{docname}.rst"
+    if not rst_path.is_file() or not source[0].strip():
+        return
+
     src: str = source[0]
     rendered: str = app.builder.templates.render_string(  # type: ignore[attr-defined]
         src,

docs/source/contribute/document.rst

@@ -390,6 +390,43 @@ Common issues
 
 .. include:: _getting_help.rst
 
+Working with tutorial notebooks
+===============================
+
+Tutorials live in ``examples/tutorials/`` as paired `Jupytext <https://jupytext.readthedocs.io/>`_
+files: a ``.py`` (percent format) source of truth and a ``.ipynb`` companion kept in sync.
+
+Key rules
+---------
+
+* **Edit the** ``.py`` **file**, not the ``.ipynb`` — the script is the single source of truth.
+* **Never commit notebook outputs.** The ``.ipynb`` on ``main`` must be output-free.
+* Notebooks are rendered into the docs via `myst-nb <https://myst-nb.readthedocs.io/>`_ with
+  cached execution (``nb_execution_mode = "cache"``).
+
+Workflow
+--------
+
+.. code-block:: bash
+
+    # After editing a .py tutorial:
+    poe notebooks          # Sync .py → .ipynb
+
+    # Before committing:
+    poe notebooks-clear    # Strip any outputs from .ipynb
+    poe notebooks-check    # Verify sync + no outputs (runs in CI)
+
+Creating a new tutorial
+-----------------------
+
+.. code-block:: bash
+
+    # Create the .py file in percent format, then pair it:
+    jupytext --set-formats "ipynb,py:percent" examples/tutorials/my_tutorial.py
+
+    # Add a toctree entry in docs/source/examples.rst
+    # Optionally tag the first SPDX cell with "remove-cell" (see existing tutorials)
+
 Additional documentation resources
 ==================================
 

docs/source/examples.rst

@@ -7,6 +7,12 @@
 Examples
 ========
 
-.. admonition:: This page is under construction. Will be autogenerated from examples/examples.
+End-to-end tutorials demonstrating OpenSTEF workflows. Each example is a runnable
+Jupyter notebook rendered with executed outputs.
 
-    Want to help? Check :ref:`Contributing <contributing>` for more information.
+.. toctree::
+   :maxdepth: 1
+
+   Forecasting with Presets <tutorials/forecasting_with_workflow_presets>
+   Hyperparameter Tuning <tutorials/hyperparameter_tuning_with_optuna>
+   Backtesting with BEAM <tutorials/backtesting_openstef_with_beam>

examples/tutorials/backtesting_openstef_with_beam.ipynb

@@ -37,6 +37,7 @@
     "# --- Thread Configuration ---\n",
     "# Prevent thread contention when running parallel backtests with XGBoost\n",
     "import os\n",
+    "\n",
     "os.environ[\"OMP_NUM_THREADS\"] = \"1\"\n",
     "os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\"\n",
     "os.environ[\"MKL_NUM_THREADS\"] = \"1\"\n",
@@ -70,8 +71,8 @@
    "outputs": [],
    "source": [
     "# Import types for configuration\n",
-    "from openstef_core.types import LeadTime, Q  # LeadTime: forecast horizon, Q: quantile\n",
     "from openstef_beam.benchmarking.benchmarks.liander2024 import Liander2024Category\n",
+    "from openstef_core.types import LeadTime, Q  # LeadTime: forecast horizon, Q: quantile\n",
     "\n",
     "# --- Output Paths ---\n",
     "OUTPUT_PATH = Path(\"./benchmark_results\")\n",
@@ -87,9 +88,13 @@
     "\n",
     "# Quantiles for probabilistic forecasting (7 quantiles covering 5th to 95th percentile)\n",
     "PREDICTION_QUANTILES = [\n",
-    "    Q(0.05), Q(0.1), Q(0.3),  # Lower quantiles\n",
-    "    Q(0.5),                    # Median\n",
-    "    Q(0.7), Q(0.9), Q(0.95),  # Upper quantiles\n",
+    "    Q(0.05),\n",
+    "    Q(0.1),\n",
+    "    Q(0.3),  # Lower quantiles\n",
+    "    Q(0.5),  # Median\n",
+    "    Q(0.7),\n",
+    "    Q(0.9),\n",
+    "    Q(0.95),  # Upper quantiles\n",
     "]\n",
     "\n",
     "# --- Benchmark Filter (optional) ---\n",
@@ -128,26 +133,21 @@
     "    model_id=\"benchmark_model_\",\n",
     "    run_name=None,\n",
     "    model=\"flatliner\",  # Placeholder - will be overwritten per model\n",
-    "    \n",
     "    # Forecast settings\n",
     "    horizons=FORECAST_HORIZONS,\n",
     "    quantiles=PREDICTION_QUANTILES,\n",
-    "    \n",
     "    # Model reuse: reuse trained model for same target (speeds up backtesting)\n",
     "    model_reuse_enable=True,\n",
     "    mlflow_storage=None,  # Disable MLflow for this demo\n",
-    "    \n",
     "    # Weather feature column mappings (match dataset column names)\n",
     "    radiation_column=\"shortwave_radiation\",\n",
     "    wind_speed_column=\"wind_speed_80m\",  # 80m wind speed for better wind park predictions\n",
     "    pressure_column=\"surface_pressure\",\n",
     "    temperature_column=\"temperature_2m\",\n",
     "    relative_humidity_column=\"relative_humidity_2m\",\n",
-    "    \n",
     "    # Additional features\n",
     "    energy_price_column=\"EPEX_NL\",  # Day-ahead electricity price\n",
     "    rolling_aggregate_features=[\"mean\", \"median\", \"max\", \"min\"],  # Rolling window stats\n",
-    "    \n",
     "    # Logging\n",
     "    verbosity=0,  # Quiet mode for batch processing\n",
     ")"
@@ -228,9 +228,9 @@
    "outputs": [],
    "source": [
     "# Import benchmark components\n",
+    "from openstef_beam.benchmarking.baselines.openstef4 import create_openstef4_preset_backtest_forecaster\n",
     "from openstef_beam.benchmarking.benchmarks.liander2024 import create_liander2024_benchmark_runner\n",
     "from openstef_beam.benchmarking.callbacks.strict_execution_callback import StrictExecutionCallback\n",
-    "from openstef_beam.benchmarking.baselines.openstef4 import create_openstef4_preset_backtest_forecaster\n",
     "\n",
     "# --- Run XGBoost Benchmark ---\n",
     "print(\"🌲 Running XGBoost benchmark...\")\n",
@@ -300,10 +300,12 @@
     "\n",
     "# Generate comparison reports\n",
     "print(\"📊 Generating comparison analysis...\")\n",
-    "comparison_pipeline.run(run_data={\n",
-    "    \"xgboost\": storage_xgboost,\n",
-    "    \"gblinear\": storage_gblinear,\n",
-    "})\n",
+    "comparison_pipeline.run(\n",
+    "    run_data={\n",
+    "        \"xgboost\": storage_xgboost,\n",
+    "        \"gblinear\": storage_gblinear,\n",
+    "    }\n",
+    ")\n",
     "print(\"✅ Comparison analysis complete!\")"
    ]
   },
@@ -331,11 +333,11 @@
    "source": [
     "# Open key analysis plots in browser\n",
     "# HTML visualizations are interactive and best viewed in a browser\n",
-    "import webbrowser\n",
     "import os\n",
+    "import webbrowser\n",
     "\n",
     "# Base path for analysis results\n",
-    "analysis_base = os.path.abspath('./benchmark_results/analysis/D-1T06:00')\n",
+    "analysis_base = os.path.abspath(\"./benchmark_results/analysis/D-1T06:00\")\n",
     "\n",
     "# Define key visualizations to open\n",
     "visualizations = [\n",
@@ -346,9 +348,9 @@
     "print(\"🌐 Opening analysis visualizations in browser...\\n\")\n",
     "for name, filename in visualizations:\n",
     "    filepath = os.path.join(analysis_base, filename)\n",
-    "    if os.path.exists(filepath):\n",
+    "    if Path(filepath).exists():\n",
     "        print(f\"   📊 {name}\")\n",
-    "        webbrowser.open(f'file://{filepath}')\n",
+    "        webbrowser.open(f\"file://{filepath}\")\n",
     "    else:\n",
     "        print(f\"   ⚠️  {name} not found at {filepath}\")"
    ]
@@ -374,22 +376,22 @@
     "import glob\n",
     "\n",
     "# Find all time series plots for individual targets\n",
-    "target_plots = glob.glob('./benchmark_results/XGBoost/analysis/*/*/time_series_plot*.html')\n",
+    "target_plots = glob.glob(\"./benchmark_results/XGBoost/analysis/*/*/time_series_plot*.html\")\n",
     "\n",
     "if target_plots:\n",
     "    print(\"📊 Available target-specific time series plots:\\n\")\n",
     "    for i, plot in enumerate(sorted(target_plots)[:5]):  # Show first 5\n",
-    "        parts = plot.split('/')\n",
+    "        parts = plot.split(\"/\")\n",
     "        category = parts[-3]  # e.g., \"transformer\"\n",
-    "        target = parts[-2]    # e.g., \"OS Apeldoorn\"\n",
-    "        print(f\"   {i+1}. {category}/{target}\")\n",
-    "    \n",
+    "        target = parts[-2]  # e.g., \"OS Apeldoorn\"\n",
+    "        print(f\"   {i + 1}. {category}/{target}\")\n",
+    "\n",
     "    # Open the first transformer plot as an example\n",
-    "    transformer_plots = [p for p in target_plots if 'transformer' in p]\n",
+    "    transformer_plots = [p for p in target_plots if \"transformer\" in p]\n",
     "    if transformer_plots:\n",
     "        example_plot = os.path.abspath(transformer_plots[0])\n",
     "        print(f\"\\n🌐 Opening example: {transformer_plots[0]}\")\n",
-    "        webbrowser.open(f'file://{example_plot}')\n",
+    "        webbrowser.open(f\"file://{example_plot}\")\n",
     "else:\n",
     "    print(\"⚠️  No target-specific plots found. Run the benchmark first.\")"
    ]
@@ -437,6 +439,9 @@
   }
  ],
  "metadata": {
+  "jupytext": {
+   "formats": "ipynb,py:percent"
+  },
   "kernelspec": {
    "display_name": ".venv",
    "language": "python",