diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..bd337d77 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "py_src/ipyelk/tests/elk-models"] + path = py_src/ipyelk/tests/elk-models + url = https://github.com/eclipse/elk-models.git diff --git a/.prettierignore b/.prettierignore index 4cb93cee..06e3d8d1 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,7 +2,10 @@ **/__pycache__/**/* **/.ipynb_checkpoints/**/* build/ +dist envs/ lib/ node_modules/ py_src/ipyelk/labextension +py_src/ipyelk/tests/elk-models +py_src/ipyelk/tests/fixtures diff --git a/MANIFEST.in b/MANIFEST.in index 8abe22e5..51f6f3aa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ recursive-include py_src/ipyelk/labextension *.* recursive-exclude scripts *.* exclude src lib docs examples package.json scripts - +prune py_src/ipyelk/tests/elk-models global-exclude *~ global-exclude *.pyc global-exclude *.pyo diff --git a/dodo.py b/dodo.py index c5b9a79a..be318524 100644 --- a/dodo.py +++ b/dodo.py @@ -19,6 +19,7 @@ from doit.action import CmdAction from doit.tools import LongRunning, PythonInteractiveAction, config_changed +from scripts import migrate_models as M from scripts import project as P from scripts import reporter from scripts import utils as U @@ -232,6 +233,17 @@ def task_setup(): ) +def task_fixtures(): + """migrate elk-models to fixtures""" + return dict( + file_dep=[P.SCRIPTS / "migrate_models.py", *M.ELKMODEL_ELKT, *M.ELKMODEL_JSON], + actions=[M.migrate], + uptodate=[False], + targets=["fake_target_since_somehow_duplicating_doit_task"], + # targets=[f for f in M.ELKMODEL_FIXTURES], + ) + + if not P.TESTING_IN_CI: def task_build(): diff --git a/examples/14_elk_model_explorer.ipynb b/examples/14_elk_model_explorer.ipynb new file mode 100644 index 00000000..5ad650c3 --- /dev/null +++ b/examples/14_elk_model_explorer.ipynb @@ -0,0 +1,195 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🦌 Elk Model Explorer ⚡\n", + "\n", + "The [Elk Model Repositoy](https://github.com/eclipse/elk-models) was converted to elk json and included as a set of fixtures to test against. This notebook loads those fixtures for users to explore to understand what kinds of layout variety possible as well and see the originating elk json." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import asyncio\n", + "\n", + "from pathlib import Path\n", + "from IPython.display import display, JSON\n", + "from ipyelk import ElkDiagram, tests\n", + "from ipyelk.diagram.elk_text_sizer import size_labels, ElkTextSizer\n", + "from ipyelk.diagram.elk_model import ElkLabel\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ran = []\n", + "async def size_labels(text_sizer, labels):\n", + " ran.append(labels)\n", + " sizes = await text_sizer.measure(tuple(ElkLabel.from_dict(l) for l in labels))\n", + " \n", + " for size, label in zip(sizes, labels):\n", + " label[\"width\"] = size.width\n", + " label[\"height\"] = size.height\n", + " \n", + " \n", + "def collect_labels(data):\n", + " \n", + " labels = []\n", + " labels += data.get(\"labels\", [])\n", + " for prop in [\"ports\", \"children\", \"edges\"]:\n", + " for value in data.get(prop, []):\n", + " labels += collect_labels(value)\n", + " return labels\n", + "\n", + "def default_node_size(data, width=40, height=40):\n", + " if \"width\" not in data:\n", + " data[\"width\"] = width\n", + " if \"height\" not in data:\n", + " data[\"height\"] = height\n", + " for child in data.get(\"children\", []):\n", + " default_node_size(child)\n", + "\n", + "def fixture_explorer():\n", + " #TODO some labels do not have a width / height provided ... could use the text sizer widget and size labels accordingly\n", + " #TODO some nodes do not have a specified width / height ... set a default\n", + " fixtures = Path(tests.__file__).parent / \"fixtures\"\n", + " fixtures\n", + "\n", + " options = [f.relative_to(fixtures) for f in fixtures.rglob(\"*.json\")]\n", + " len(options)\n", + " import ipywidgets as W\n", + " diagram = ElkDiagram(layout={\"flex\":\"1\"})\n", + " selector = W.Dropdown(options=options)\n", + " err_btn = W.Button()\n", + " output = W.Output()\n", + " text_sizer = ElkTextSizer(max_size=20)\n", + "\n", + " def set_err(message):\n", + " if message:\n", + " err_btn.icon = \"warning\"\n", + " err_btn.tooltip = message\n", + " else:\n", + " err_btn.icon = \"check\"\n", + " err_btn.tooltip = \"\"\n", + "\n", + " \n", + " async def _load():\n", + " # load fixture elk json\n", + " if not selector.value:\n", + " return\n", + " \n", + " path = (fixtures / selector.value)\n", + " if not path.exists():\n", + " return\n", + " data = json.loads(path.read_text())\n", + " \n", + " # add width and height to labels\n", + " await size_labels(text_sizer, collect_labels(data))\n", + " \n", + " # add default size to nodes\n", + " default_node_size(data)\n", + " \n", + " try:\n", + " diagram.value = {\"id\":\"root\"} # needed for now to clear sprotty diagram. until fixed https://github.com/jupyrdf/ipyelk/issues/17\n", + " diagram.value = data\n", + " set_err(\"\")\n", + " except Exception as e:\n", + " diagram.value = {\"id\":\"root\"}\n", + " set_err(str(e))\n", + " \n", + " def update(change=None):\n", + " asyncio.create_task(_load())\n", + " \n", + " def refresh_json_view(change=None):\n", + " output.clear_output()\n", + " with output:\n", + " display(JSON(diagram.value))\n", + " \n", + " \n", + " diagram.observe(refresh_json_view, \"value\")\n", + "\n", + "\n", + " model_explorer = W.VBox(children=[\n", + "\n", + " W.HBox(\n", + " children=[\n", + " selector,\n", + " err_btn,\n", + " ]\n", + " ),\n", + " W.HBox(\n", + " children=[\n", + " diagram,\n", + " output,\n", + " ],\n", + " layout={\"flex\":\"1\"})\n", + "\n", + " ],\n", + " layout={\"height\":\"100%\"} \n", + " )\n", + " selector.observe(update, \"value\")\n", + " update()\n", + " return model_explorer, diagram, output, refresh_json_view\n", + "\n", + "if __name__ == \"__main__\":\n", + " explorer, diagram, output, refresh_json_view = fixture_explorer()\n", + " display(explorer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO understand why this refresh_json_view function works if called manually but no longer seems to update the output view using the observers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "refresh_json_view()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/_index.ipynb b/examples/_index.ipynb index bfa1312f..80051907 100644 --- a/examples/_index.ipynb +++ b/examples/_index.ipynb @@ -14,7 +14,8 @@ "## Basic Examples\n", "\n", "- [🦌 Introducing ELK 👋](./00_Introduction.ipynb)\n", - "- [🦌 Linking ELK Diagrams 🔗](./01_Linking.ipynb)" + "- [🦌 Linking ELK Diagrams 🔗](./01_Linking.ipynb)\n", + "- [🦌 ELK Models Explorer 🔗](./14_elk_model_explorer.ipynb)" ] }, { diff --git a/py_src/ipyelk/tests/elk-models b/py_src/ipyelk/tests/elk-models new file mode 160000 index 00000000..e46a263a --- /dev/null +++ b/py_src/ipyelk/tests/elk-models @@ -0,0 +1 @@ +Subproject commit e46a263a86fbde594d8eb25374b0852117a0d94e diff --git a/scripts/migrate_models.py b/scripts/migrate_models.py new file mode 100644 index 00000000..a1c9c493 --- /dev/null +++ b/scripts/migrate_models.py @@ -0,0 +1,111 @@ +""" Convert [Elk Model Repository](https://github.com/eclipse/elk-models) into ElkJSON +""" + +# Copyright (c) 2021 Dane Freeman. +# Distributed under the terms of the Modified BSD License. + +import json +from itertools import chain +from pathlib import Path +from typing import Dict +from uuid import uuid4 + +import requests + +from .project import ELKFIXTURES, ELKMODELS + +ELKMODEL_ELKT = [f for f in ELKMODELS.rglob("*.elkt")] +ELKMODEL_JSON = [f for f in ELKMODELS.rglob("*.json")] + + +def fixture(model: Path) -> Path: + """Get mapped fixture target + + :param model: model path + :return: Elk fixture that should exist after migrating models + """ + return ELKFIXTURES / model.relative_to(ELKMODELS).with_suffix(".json") + + +ELKMODEL_FIXTURES = [fixture(f) for f in chain(ELKMODEL_ELKT, ELKMODEL_JSON)] + + +def migrate_layout_options(data: Dict) -> Dict: + """The older klayjs json uses the `properties` key which needs to be + remapped to `layoutOptions` + + :param data: klayjs JSON + :return: Updated ElkJSON + """ + data = {**data} + layout_options = data.pop("properties", None) + if layout_options: + data["layoutOptions"] = layout_options + + for prop in ["ports", "children", "labels"]: + value = [migrate_layout_options(d) for d in data.get(prop, [])] + if value: + data[prop] = value + return data + + +def backfill_ids(data: Dict) -> Dict: + """Seems like some `id`s are missing. This will backfill as needed + + :param data: JSON + :return: Updated ElkJSON + """ + data = {**data} + data["id"] = data.get("id", str(uuid4())) + + for prop in ["ports", "children", "labels"]: + value = [backfill_ids(d) for d in data.get(prop, [])] + if value: + data[prop] = value + + edges = [] + for edge in data.get("edges", []): + edges.append(backfill_ids(edge)) + if edges: + data["edges"] = edges + return data + + +def elkt_to_elkjson(data: str) -> Dict: + """Uses public server to convert elkt to elk json + + :param data: elkt text + :return: ElkJSON + """ + # TODO maybe this url can be configured elsewhere but it isn't immediately + # discoverable + url = "https://rtsys.informatik.uni-kiel.de/elklive/conversion" + params = {"inFormat": "elkt", "outFormat": "json"} + headers = { + "Content-Type": "text/plain", + } + resp = requests.post(url, params=params, data=data.encode("utf-8"), headers=headers) + if resp.status_code == 200: + return json.loads(resp.content.decode("utf-8")) + + +def migrate(force=False): + """Migrate elk-models' elkt and older json formats to fixtures""" + ELKFIXTURES.mkdir(parents=True, exist_ok=True) + + def save(model, elkjson): + elkjson = backfill_ids(elkjson) + path = fixture(model) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(elkjson)) + + for model in ELKMODEL_JSON: + elkjson = migrate_layout_options(json.loads(model.read_text())) + save(model, elkjson) + + for model in ELKMODEL_ELKT: + path = fixture(model) + if force or not path.exists(): + elkjson = elkt_to_elkjson(model.read_text()) + + save(model, elkjson) diff --git a/scripts/project.py b/scripts/project.py index 1dad6549..6a4e9fb4 100644 --- a/scripts/project.py +++ b/scripts/project.py @@ -132,9 +132,11 @@ EXAMPLE_PY = [*EXAMPLES.rglob("*.py")] EXAMPLE_INDEX = EXAMPLES / "_index.ipynb" BUILD_NBHTML = BUILD / "nbsmoke" +ELKMODELS = PY_SRC / "tests" / "elk-models" +ELKFIXTURES = PY_SRC / "tests" / "fixtures" # mostly linting -ALL_PY_SRC = [*PY_SRC.rglob("*.py")] +ALL_PY_SRC = [p for p in PY_SRC.rglob("*.py") if str(ELKMODELS) not in str(p)] ALL_PY = [ *ALL_PY_SRC, *EXAMPLE_PY,