From a36809b2566c2c10a8f203694ab32b5888d57947 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 25 Mar 2026 14:23:28 +0100 Subject: [PATCH 01/15] Add anywidget backend --- examples/rendercanvas.ipynb | 6 +- rendercanvas/anywidget.py | 343 ++++++++++++++++++++++ rendercanvas/core/renderview-anywidget.js | 249 ++++++++++++++++ rendercanvas/jupyter.py | 2 +- 4 files changed, 597 insertions(+), 3 deletions(-) create mode 100644 rendercanvas/anywidget.py create mode 100644 rendercanvas/core/renderview-anywidget.js diff --git a/examples/rendercanvas.ipynb b/examples/rendercanvas.ipynb index 7ef3679b..3bf79b70 100644 --- a/examples/rendercanvas.ipynb +++ b/examples/rendercanvas.ipynb @@ -24,7 +24,9 @@ "outputs": [], "source": [ "from rendercanvas.utils.cube import setup_drawing_sync\n", - "from rendercanvas.jupyter import RenderCanvas\n", + "\n", + "# from rendercanvas.jupyter import RenderCanvas # Uses jupyter_rfb\n", + "from rendercanvas.anywidget import RenderCanvas # Uses 'native' anywidget backend\n", "\n", "canvas = RenderCanvas(update_mode=\"continuous\")\n", "draw_frame = setup_drawing_sync(canvas)\n", @@ -76,7 +78,7 @@ { "cell_type": "code", "execution_count": null, - "id": "65f86895-9e9a-4c22-8a54-919bd70fd80b", + "id": "f986c9c5-e803-4789-b9a8-d094e44a3040", "metadata": {}, "outputs": [], "source": [] diff --git a/rendercanvas/anywidget.py b/rendercanvas/anywidget.py new file mode 100644 index 00000000..70f4930c --- /dev/null +++ b/rendercanvas/anywidget.py @@ -0,0 +1,343 @@ +""" +A backend based on anywidget, supporting canvases inside a large variety of notebooks and similar browser-like environments. +""" + +__all__ = ["AnywidgetRenderCanvas", "RenderCanvas", "loop"] + +import time +import asyncio +from base64 import encodebytes +from importlib.resources import files as resource_files + +from .base import BaseCanvasGroup, BaseRenderCanvas, logger +from .asyncio import loop +from .core.encoders import encode_array, CAN_JPEG + +import numpy as np +import anywidget +from traitlets import Bool, Dict, Int, Unicode + + +def _load_js_and_css(): + js = "" + for fname in ["renderview.js", "renderview-anywidget.js"]: + js_path = resource_files("rendercanvas.core").joinpath(fname) + js += js_path.read_text() + "\n\n" + + css_path = resource_files("rendercanvas.core").joinpath("renderview.css") + + return js, css_path.read_text() + + +JS, CSS = _load_js_and_css() + + +class AnywidgetCanvasGroup(BaseCanvasGroup): + pass + + +class AnywidgetRenderCanvas(BaseRenderCanvas, anywidget.AnyWidget): + """An anywidget canvas to use in notebooks (e.g. jupyter, marimo, VSCode, etc.). + + This is an AnyWidget subclass, so you can easily combine it with other widgets. + """ + + # This class uses some '_rfb_' prefixes to avoid name clashes with super and sub classes. + # This specific prefix was inherited from jupyter_rfb, and we decided to keep it as is. + + _rc_canvas_group = AnywidgetCanvasGroup(loop) + + _esm = JS + _css = CSS + + # Client -> server + _frame_feedback = Dict({}).tag(sync=True) + _has_visible_views = Bool(False).tag(sync=True) + # Server -> client + _css_width = Unicode("500px").tag(sync=True) + _css_height = Unicode("300px").tag(sync=True) + _resizable = Bool(True).tag(sync=True) + _has_titlebar = Bool(False).tag(sync=True) + _title = Unicode("").tag(sync=True) + _cursor = Unicode("default").tag(sync=True) + # Server only + _max_buffered_frames = Int(2, min=1) + _quality = Int(80, min=1, max=100) + + def __init__(self, *args, **kwargs): + # This backend's default title is empty + kwargs["title"] = kwargs.get("title", "") + + super().__init__(*args, **kwargs) + + self._is_closed = False + + self._rfb_draw_requested = False + self._rfb_frame_index = 0 + self._rfb_last_confirmed_index = 0 + self._rfb_warned_png = False + self._rfb_lossless_draw_info = None + self._use_websocket = True # Could be a prop, private for now + + self.reset_stats() + self.on_msg(self._rfb_handle_msg) + self.observe( + self._rfb_schedule_maybe_draw, + names=["_frame_feedback", "_has_visible_views"], + ) + + # Set size, title, etc. + self._final_canvas_init() + + def _rfb_handle_msg(self, widget, content, buffers): + """Receive custom messages and filter our events.""" + event_type = content.get("type") + + if event_type is not None: + event = content + + if event_type == "resize": + self._last_event = event + self._size_info.set_physical_size( + event["pwidth"], event["pheight"], event["ratio"] + ) + elif event_type == "close": + self.close() + else: + # Compatibility between new renderview event spec and current rendercanvas/pygfx events + event["event_type"] = event.pop("type") + event["time_stamp"] = event.pop("timestamp") + # Turn lists into tuples (js/json does not have tuples) + if "buttons" in event: + event["buttons"] = tuple(event["buttons"]) + if "modifiers" in event: + event["modifiers"] = tuple(event["modifiers"]) + self.submit_event(event) + + def _rfb_schedule_maybe_draw(self, *args): + """Schedule _maybe_draw() to be called in a fresh event loop iteration.""" + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + loop.call_soon(self._rfb_maybe_draw) + + def _rfb_maybe_draw(self): + """Perform a draw, if we can and should.""" + feedback = self._frame_feedback + # Update stats + self._rfb_update_stats(feedback) + # Determine whether we should perform a draw: a draw was requested, and + # the client is ready for a new frame, and the client widget is visible. + frames_in_flight = self._rfb_frame_index - feedback.get("index", 0) + should_draw = ( + self._rfb_draw_requested + and frames_in_flight < self._max_buffered_frames + and self._has_visible_views + ) + # Do the draw if we should. + if should_draw: + self._rfb_draw_requested = False + self._time_to_draw() # -> _rc_present_bitmap -> _rfb_send_frame + + def _rfb_schedule_lossless_draw(self, array, delay=0.3): + self._rfb_cancel_lossless_draw() + loop = asyncio.get_running_loop() + handle = loop.call_later(delay, self._rfb_lossless_draw) + self._rfb_lossless_draw_info = array, handle + + def _rfb_cancel_lossless_draw(self): + if self._rfb_lossless_draw_info: + _, handle = self._rfb_lossless_draw_info + self._rfb_lossless_draw_info = None + handle.cancel() + + def _rfb_lossless_draw(self): + array, _ = self._rfb_lossless_draw_info + self._rfb_send_frame(array, True) + + def _rfb_send_frame(self, array, is_lossless_redraw=False): + """Actually send a frame over to the client.""" + # For considerations about performance, + # see https://github.com/vispy/jupyter_rfb/issues/3 + + quality = 100 if is_lossless_redraw else self._quality + + self._rfb_frame_index += 1 + timestamp = time.time() + + # Turn array into a based64-encoded JPEG or PNG + t1 = time.perf_counter() + mimetype, data = encode_array(array, quality) + if self._use_websocket: + datas = [data] + data_b64 = None + else: + datas = [] + data_b64 = f"data:{mimetype};base64," + encodebytes(data).decode() + t2 = time.perf_counter() + + if "jpeg" in mimetype: + self._rfb_schedule_lossless_draw(array) + else: + self._rfb_cancel_lossless_draw() + # Issue png warning? + if quality < 100 and not CAN_JPEG and not self._rfb_warned_png: + self._rfb_warned_png = True + logger.warning( + "No JPEG encoder found, using PNG instead. Install simplejpeg for better performance." + ) + + if is_lossless_redraw: + # No stats, also not on the confirmation of this frame + self._rfb_last_confirmed_index = self._rfb_frame_index + else: + # Stats + self._rfb_stats["img_encoding_sum"] += t2 - t1 + self._rfb_stats["sent_frames"] += 1 + if self._rfb_stats["start_time"] <= 0: # Start measuring + self._rfb_stats["start_time"] = timestamp + self._rfb_last_confirmed_index = self._rfb_frame_index - 1 + + # Compose message and send + msg = dict( + type="framebufferdata", + mimetype=mimetype, + data_b64=data_b64, + index=self._rfb_frame_index, + timestamp=timestamp, + ) + self.send(msg, datas) + + # ----- related to stats + + def reset_stats(self): + """Restart measuring statistics from the next sent frame.""" + self._rfb_stats = { + "start_time": 0, + "last_time": 1, + "sent_frames": 0, + "confirmed_frames": 0, + "roundtrip_count": 0, + "roundtrip_sum": 0, + "delivery_sum": 0, + "img_encoding_sum": 0, + } + + def get_stats(self): + """Get the current stats since the last time ``.reset_stats()`` was called. + + Stats is a dict with the following fields: + + * *sent_frames*: the number of frames sent. + * *confirmed_frames*: number of frames confirmed by the client. + * *roundtrip*: average time for processing a frame, including receiver confirmation. + * *delivery*: average time for processing a frame until it's received by the client. + This measure assumes that the clock of the server and client are precisely synced. + * *img_encoding*: the average time spent on encoding the array into an image. + * *b64_encoding*: the average time spent on base64 encoding the data. + * *fps*: the average FPS, measured from the first frame sent since ``.reset_stats()`` + was called, until the last confirmed frame. + """ + d = self._rfb_stats + roundtrip_count_div = d["roundtrip_count"] or 1 + sent_frames_div = d["sent_frames"] or 1 + fps_div = (d["last_time"] - d["start_time"]) or 0.001 + return { + "sent_frames": d["sent_frames"], + "confirmed_frames": d["confirmed_frames"], + "roundtrip": d["roundtrip_sum"] / roundtrip_count_div, + "delivery": d["delivery_sum"] / roundtrip_count_div, + "img_encoding": d["img_encoding_sum"] / sent_frames_div, + "fps": d["confirmed_frames"] / fps_div, + } + + def _rfb_update_stats(self, feedback): + """Update the stats when a new frame feedback has arrived.""" + last_index = feedback.get("index", 0) + if last_index > self._rfb_last_confirmed_index: + timestamp = feedback["timestamp"] + nframes = last_index - self._rfb_last_confirmed_index + self._rfb_last_confirmed_index = last_index + self._rfb_stats["confirmed_frames"] += nframes + self._rfb_stats["roundtrip_count"] += 1 + self._rfb_stats["roundtrip_sum"] += time.time() - timestamp + self._rfb_stats["delivery_sum"] += feedback["localtime"] - timestamp + self._rfb_stats["last_time"] = time.time() + + # --- the API to be a rendercanvas backend + + def _rc_gui_poll(self): + pass + + def _rc_get_present_info(self, present_methods): + # Only allow simple format for now. srgb is assumed. + if "bitmap" in present_methods: + return { + "method": "bitmap", + "formats": ["rgba-u8"], + } + else: + return None # raises error + + def _rc_request_draw(self): + # Technically, _maybe_draw() may not perform a draw if there are too + # many frames in-flight. But in this case, we'll eventually get + # new frame_feedback, which will then trigger a draw. + if not self._rfb_draw_requested: + self._rfb_draw_requested = True + self._rfb_cancel_lossless_draw() + self._rfb_schedule_maybe_draw() + + def _rc_request_paint(self): + # We technically don't need to call _time_to_paint, because this backend only does bitmap mode. + # But in case the base backend will do something in _time_to_paint later, we behave nice. + loop = self._rc_canvas_group.get_loop() + loop.call_soon(self._time_to_paint) + + def _rc_force_paint(self): + pass # works as-is via push_frame + + def _rc_present_bitmap(self, *, data, format, **kwargs): + assert format == "rgba-u8" + self._rfb_send_frame(np.asarray(data)) + + def _rc_set_logical_size(self, width, height): + self._css_width = f"{width}px" + self._css_height = f"{height}px" + + def _rc_close(self): + anywidget.AnyWidget.close(self) + self._rfb_handle_msg(self, {"type": "close"}, []) + self._is_closed = True + + def _rc_get_closed(self): + return self._is_closed + + def _rc_set_title(self, title): + self._title = str(title) + self._has_titlebar = bool(title) + + def _rc_set_cursor(self, cursor): + self._cursor = cursor + + def set_css_width(self, css_width: str): + """Set the width of the canvas as a CSS string.""" + self._css_width = css_width + + def set_css_height(self, css_height: str): + """Set the height of the canvas as a CSS string.""" + self._css_height = css_height + + def set_resizable(self, resizable: bool): + """Set whether the canvas is manually resizable. + + Note that the canvas can only be made resizable if it was attached to a + wrapper HTML element (not directly to a ````). + """ + self._resizable = resizable + + +# Make available under a common name +RenderCanvas = AnywidgetRenderCanvas +loop = loop diff --git a/rendercanvas/core/renderview-anywidget.js b/rendercanvas/core/renderview-anywidget.js new file mode 100644 index 00000000..ded002b3 --- /dev/null +++ b/rendercanvas/core/renderview-anywidget.js @@ -0,0 +1,249 @@ +/* global BaseRenderView getTimestamp */ + +/** + * An object that represents the model (wrapping the anywidget model object), that can have multiple views. + */ +class RendercanvasAnywidgetModel { + constructor (anymodel) { + this.anymodel = anymodel + this.views = [] + this._hasVisibleViews = false + + // Variables to store frames and the last frame + this._frames = [] + this._lastSrc = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mOor68HAAL+AX6E2KOJAAAAAElFTkSuQmCC' + this._lastFrame = { + src: this._lastSrc, + index: 0, + timestamp: 0 + } + + // Register callbacks + anymodel.on('msg:custom', (msg, buffers) => { + if (msg.type === 'framebufferdata') { + this._frames.push({ ...msg, buffers }) + this._request_animation_frame() + } + }) + // For traits we allow the public and private version, this means we can use the + // same JS code for multiple anywidget implementations (specifically rendercanvas.anywidget vs jupyter_rfb) + for (const prefix of ['', '_']) { + anymodel.on(`change:${prefix}css_width`, () => { + const cssWidth = anymodel.get(`${prefix}css_width`) + for (const view of this.views) { + view.setCssWidth(cssWidth) + } + }) + anymodel.on(`change:${prefix}css_height`, () => { + const cssHeight = anymodel.get(`${prefix}css_height`) + for (const view of this.views) { + view.setCssHeight(cssHeight) + } + }) + anymodel.on(`change:${prefix}resizable`, () => { + const resizable = anymodel.get(`${prefix}resizable`) + for (const view of this.views) { + view.setResizable(resizable) + } + }) + anymodel.on(`change:${prefix}has_titlebar`, () => { + const titlebar = anymodel.get(`${prefix}has_titlebar`) + for (const view of this.views) { + view.showTitlebar(titlebar) + } + }) + anymodel.on(`change:${prefix}title`, () => { + const title = anymodel.get(`${prefix}title`) + for (const view of this.views) { + view.setTitle(title) + } + }) + anymodel.on(`change:${prefix}cursor`, () => { + const cursor = anymodel.get(`${prefix}cursor`) + for (const view of this.views) { + view.setCursor(cursor) + } + }) + } + + // Start the animation loop + this._img_update_pending = false + this._request_animation_frame() + } + + close () { + URL.revokeObjectURL(this._lastSrc) + this._lastSrc = null + this._lastFrame = null + this._frames = [] + for (const view of this.views) { + view.close() + } + // This gets called when the model is closed and the comm is removed. Notify Py just in time! + const t = getTimestamp() + const event = { + type: 'close', + timestamp: t + } + this.onEvent(event) + } + + addView (view) { + this.views.push(view) + this.updateVisibility() + // Init attrs + const anymodel = this.anymodel + view.setCssWidth(anymodel.get('_css_width') ?? anymodel.get('css_width')) + view.setCssHeight(anymodel.get('_css_height') ?? anymodel.get('css_height')) + view.setResizable(anymodel.get('_resizable') ?? anymodel.get('resizable')) + view.showTitlebar(anymodel.get('_has_titlebar') ?? anymodel.get('has_titlebar')) + view.setTitle(anymodel.get('_title') ?? anymodel.get('title')) + view.setCursor(anymodel.get('_cursor') ?? anymodel.get('cursor')) + // Init view + if (this._lastSrc) { + view.viewElement.src = this._lastSrc + } + } + + removeView (view) { + this.views = this.views.filter(v => v !== view) + this.updateVisibility() + } + + updateVisibility () { + let visibleViewsCount = 0 + for (const view of this.views) { + if (view.isVisible) { visibleViewsCount += 1 } + } + const hasVisibleViews = visibleViewsCount > 0 + this.anymodel.set('_has_visible_views', hasVisibleViews) + this.anymodel.save_changes() + if (hasVisibleViews) { + this._request_animation_frame() + } + } + + _send_response () { + // Let Python know what we have at the frame. This prop is a dict, making it "atomic". + const frame = this._lastFrame + const frameFeedback = { index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } + this.anymodel.set('_frame_feedback', frameFeedback) + this.anymodel.save_changes() + } + + _request_animation_frame () { + // Request an animation frame. + // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, + // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. + if (!this._img_update_pending) { + this._img_update_pending = true + window.requestAnimationFrame(this._animate.bind(this)) + // window.setTimeout(window.requestAnimationFrame, 5, this._animate.bind(this)) // via a delay + } + } + + _animate () { + this._img_update_pending = false + if (this._frames.length === 0) { return }; + + // Pick the oldest frame from the stack, and get its source + const frame = this._frames.shift() + let newSrc + if (frame.buffers.length > 0) { + const blob = new Blob([frame.buffers[0].buffer], { type: frame.mimetype }) + newSrc = URL.createObjectURL(blob) + } else { + newSrc = frame.data_b64 + } + + // Revoke last objectURL + URL.revokeObjectURL(this._lastSrc) + this._lastSrc = newSrc + + // Update the image sources + for (const view of this.views) { + view.viewElement.src = newSrc + view.viewElement.onload = this._request_animation_frame.bind(this) + } + + // Let the server know we processed the image (even if it's not shown yet) + this._lastFrame = frame + this._send_response() + } + + onEvent (event) { + try { + this.anymodel.send(event) + } catch { } // probably attempt to send when widget is closed + } +} + +/** + * View to show the anywidget output and observe events, based on renderview.js. + */ +class AnywidgetRenderView extends BaseRenderView { + constructor (model, containerElement) { + // Create the wrapper element + const wrapperElement = document.createElement('div') + wrapperElement.classList.add('renderview-wrapper') + wrapperElement.classList.add('is-resizable') + // wrapperElement.classList.add('has-titlebar') -> not by default + containerElement.appendChild(wrapperElement) + + // Create img element + const viewElement = document.createElement('img') + viewElement.decoding = 'sync' + viewElement.loading = 'eager' + viewElement.style.touchAction = 'none' // prevent default pan/zoom behavior + viewElement.ondragstart = () => false // prevent browser's built-in image drag + + // Call super + super(viewElement, wrapperElement) + this.setThrottle(20) // 20ms -> max 50 move/wheel events per second + + // Connect to the model, it will initialize it with the current size and frame-data + this.model = model + this.model.addView(this) + } + + close () { + super.close() + this.model.removeView(this) + } + + onEvent (event) { + if (event.type === 'resize') { + // Note that there can be multiple views that can possibly be individually resized. + // TODO: keep logical size in check between different views? + if (event.width * event.height > 0) { + this.model.onEvent(event) + } + } else if (event.type === 'close') { + // we don't close when one view closes, only when the widget closes + } else if (event.type === 'show') { + this.isVisible = true + this.model.updateVisibility() + } else if (event.type === 'hide') { + this.isVisible = false + this.model.updateVisibility() + } else { + this.model.onEvent(event) + } + } +} + +// anywidget lifecycle export +export default () => { + let model + return { + initialize (ctx) { + model = new RendercanvasAnywidgetModel(ctx.model) + // window.model = model // debug + return () => { model.close() } + }, + render (ctx) { + const view = new AnywidgetRenderView(model, ctx.el) + return () => { view.close() } + } + } +} diff --git a/rendercanvas/jupyter.py b/rendercanvas/jupyter.py index deea7084..72e01821 100644 --- a/rendercanvas/jupyter.py +++ b/rendercanvas/jupyter.py @@ -30,7 +30,7 @@ class JupyterRenderCanvas(BaseRenderCanvas, RemoteFrameBuffer): _event_compatibility = 1 def __init__(self, *args, **kwargs): - # The jupyter backend's default title is empty + # This backend's default title is empty kwargs["title"] = kwargs.get("title", "") super().__init__(*args, **kwargs) From 0b02b003667cf050a272bbf81cbf9e5c0dd28386 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 25 Mar 2026 14:43:10 +0100 Subject: [PATCH 02/15] rename js file --- examples/rendercanvas.ipynb | 1 - rendercanvas/anywidget.py | 2 +- .../core/{renderview-anywidget.js => renderview-rfb.js} | 0 3 files changed, 1 insertion(+), 2 deletions(-) rename rendercanvas/core/{renderview-anywidget.js => renderview-rfb.js} (100%) diff --git a/examples/rendercanvas.ipynb b/examples/rendercanvas.ipynb index 3bf79b70..903cafbe 100644 --- a/examples/rendercanvas.ipynb +++ b/examples/rendercanvas.ipynb @@ -24,7 +24,6 @@ "outputs": [], "source": [ "from rendercanvas.utils.cube import setup_drawing_sync\n", - "\n", "# from rendercanvas.jupyter import RenderCanvas # Uses jupyter_rfb\n", "from rendercanvas.anywidget import RenderCanvas # Uses 'native' anywidget backend\n", "\n", diff --git a/rendercanvas/anywidget.py b/rendercanvas/anywidget.py index 70f4930c..affa92d3 100644 --- a/rendercanvas/anywidget.py +++ b/rendercanvas/anywidget.py @@ -20,7 +20,7 @@ def _load_js_and_css(): js = "" - for fname in ["renderview.js", "renderview-anywidget.js"]: + for fname in ["renderview.js", "renderview-rfb.js"]: js_path = resource_files("rendercanvas.core").joinpath(fname) js += js_path.read_text() + "\n\n" diff --git a/rendercanvas/core/renderview-anywidget.js b/rendercanvas/core/renderview-rfb.js similarity index 100% rename from rendercanvas/core/renderview-anywidget.js rename to rendercanvas/core/renderview-rfb.js From 2fda2c9f9b0948cbac492523c7e497cca62a12c7 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 25 Mar 2026 14:45:19 +0100 Subject: [PATCH 03/15] Add tests for new backend --- tests/test_backends.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_backends.py b/tests/test_backends.py index b8552f60..4d0bb280 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -256,6 +256,14 @@ def test_pyodide_module(): assert canvas_class.name == "PyodideRenderCanvas" +def test_anywidget_module(): + m = Module("anywidget") + + canvas_class = m.get_canvas_class() + m.check_canvas(canvas_class) + assert canvas_class.name == "AnywidgetRenderCanvas" + + def test_jupyter_module(): m = Module("jupyter") From b195789a4a1de5a0380f135d18bf933058c4c34e Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 25 Mar 2026 14:57:41 +0100 Subject: [PATCH 04/15] docs --- docs/backends.rst | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/backends.rst b/docs/backends.rst index 151d61fd..b49ca60e 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -20,11 +20,16 @@ The table below gives an overview of the names in the different ``rendercanvas`` | ``RenderCanvas`` (alias) | ``loop`` (an ``AsyncioLoop``) - | A lightweight backend. + * - ``anywidget`` + - | ``AnywidgetRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (an ``AsyncioLoop``) + - | Integrate in notebooks. * - ``jupyter`` - | ``JupyterRenderCanvas`` | ``RenderCanvas`` (alias) | ``loop`` (an ``AsyncioLoop``) - - | Integrate in Jupyter notebook / lab. + - | Integrate in notebooks via ``jupyter_rfb``. * - ``offscreen`` - | ``OffscreenRenderCanvas`` | ``RenderCanvas`` (alias) @@ -262,15 +267,22 @@ object, but in some cases it's convenient to do so with a canvas-like API. array = canvas.draw() # numpy array with shape (400, 500, 4) -Support for Jupyter lab and notebook ------------------------------------- +Support for notebooks +--------------------- + +RenderCanvas can be used in Jupyter lab, Jupyter notebook, VSCode, Google Colab, Marimo notebooks, and anywhere else where ``anywidget`` is supported. + +There are two backends that support the notebook: + +* The ``anywidget`` backend. +* The ``jupyter`` backend, which relies on the `jupyter_rfb `_ library. -RenderCanvas can be used in Jupyter lab and the Jupyter notebook. This canvas -is based on `jupyter_rfb `_, an ipywidget -subclass implementing a remote frame-buffer. There are also some `wgpu examples `_. +Although they share the most part of their code, the latter has some additional functionality, such as a snapshot utility. +When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected. .. code-block:: py + # from rendercanvas.anywidget import RenderCanvas # Direct approach # from rendercanvas.jupyter import RenderCanvas # Direct approach from rendercanvas.auto import RenderCanvas # also works, because rendercanvas detects Jupyter @@ -417,9 +429,8 @@ Many interactive environments have some sort of GUI support, allowing the repl to stay active (i.e. you can run new code), while the GUI windows is also alive. In rendercanvas we try to select the GUI that matches the current environment. -On ``jupyter notebook`` and ``jupyter lab`` the jupyter backend (i.e. -``jupyter_rfb``) is normally selected. When you are using ``%gui qt``, rendercanvas will -honor that and use Qt instead. +In a notebook (e.g. jupyter) one of the notebook capable backends (``anywidget`` or ``jupyter``) is selected. +When you are using ``%gui qt``, rendercanvas will honor that and use Qt instead. On ``jupyter console`` and ``qtconsole``, the kernel is the same as in ``jupyter notebook``, making it (about) impossible to tell that we cannot actually use From 8a58fd81b5c1fcff5dc80d55419b0538ef8437f3 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 25 Mar 2026 15:05:27 +0100 Subject: [PATCH 05/15] auto backend selects anywidget --- docs/backends.rst | 6 +++--- examples/rendercanvas.ipynb | 7 ++++--- rendercanvas/auto.py | 8 +++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/backends.rst b/docs/backends.rst index b49ca60e..0a9ba88a 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -282,9 +282,9 @@ When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected. .. code-block:: py - # from rendercanvas.anywidget import RenderCanvas # Direct approach - # from rendercanvas.jupyter import RenderCanvas # Direct approach - from rendercanvas.auto import RenderCanvas # also works, because rendercanvas detects Jupyter + # from rendercanvas.anywidget import RenderCanvas + # from rendercanvas.jupyter import RenderCanvas + from rendercanvas.auto import RenderCanvas # defaults to anywidget when in a notebook canvas = RenderCanvas() diff --git a/examples/rendercanvas.ipynb b/examples/rendercanvas.ipynb index 903cafbe..1a02353d 100644 --- a/examples/rendercanvas.ipynb +++ b/examples/rendercanvas.ipynb @@ -23,9 +23,10 @@ "metadata": {}, "outputs": [], "source": [ + "# from rendercanvas.anywidget import RenderCanvas\n", + "# from rendercanvas.jupyter import RenderCanvas\n", + "from rendercanvas.auto import RenderCanvas # Defaults to anywidget when in a notebook\n", "from rendercanvas.utils.cube import setup_drawing_sync\n", - "# from rendercanvas.jupyter import RenderCanvas # Uses jupyter_rfb\n", - "from rendercanvas.anywidget import RenderCanvas # Uses 'native' anywidget backend\n", "\n", "canvas = RenderCanvas(update_mode=\"continuous\")\n", "draw_frame = setup_drawing_sync(canvas)\n", @@ -77,7 +78,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f986c9c5-e803-4789-b9a8-d094e44a3040", + "id": "e8c17002-0ffa-4f11-8bd3-35a38dc5adb0", "metadata": {}, "outputs": [], "source": [] diff --git a/rendercanvas/auto.py b/rendercanvas/auto.py index 288091b5..bc52b3dd 100644 --- a/rendercanvas/auto.py +++ b/rendercanvas/auto.py @@ -20,7 +20,7 @@ # Note that wx is not in here, because it does not (yet) fully implement base.BaseRenderCanvas -BACKEND_NAMES = ["glfw", "qt", "jupyter", "offscreen"] +BACKEND_NAMES = ["glfw", "qt", "anywidget", "jupyter", "offscreen"] def _load_backend(backend_name): @@ -29,6 +29,8 @@ def _load_backend(backend_name): from . import glfw as module elif backend_name == "qt": from . import qt as module + elif backend_name == "anywidget": + from . import anywidget as module elif backend_name == "jupyter": from . import jupyter as module elif backend_name == "wx": @@ -136,7 +138,7 @@ def backends_by_notebook(): # Detect Marimo: https://github.com/marimo-team/marimo/discussions/8865 try: _ = main_module.__marimo__ - yield "jupyter", "running in Marimo" + yield "anywidget", "running in Marimo" except AttributeError: pass @@ -165,7 +167,7 @@ def backends_by_notebook(): # elif "wx" in app.__class__.__name__.lower() == "wx": # yield "wx", "running on Jupyter with wx gui" - yield "jupyter", "running on Jupyter" + yield "anywidget", "running on Jupyter" def backends_by_imported_modules(): From f4c60fe6cc6a7bb3ad69df5c7c9417f7e89907e5 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 12:17:51 +0200 Subject: [PATCH 06/15] renamd and update the js afm module --- .../{renderview-rfb.js => renderview-afm.js} | 53 +++++++++++++------ 1 file changed, 36 insertions(+), 17 deletions(-) rename rendercanvas/core/{renderview-rfb.js => renderview-afm.js} (86%) diff --git a/rendercanvas/core/renderview-rfb.js b/rendercanvas/core/renderview-afm.js similarity index 86% rename from rendercanvas/core/renderview-rfb.js rename to rendercanvas/core/renderview-afm.js index ded002b3..658b98f6 100644 --- a/rendercanvas/core/renderview-rfb.js +++ b/rendercanvas/core/renderview-afm.js @@ -1,10 +1,17 @@ +/************************************************************************************************* + renderview-afm.js + + The Anywidget Frontend Module for renderview. + + *************************************************************************************************/ + /* global BaseRenderView getTimestamp */ /** - * An object that represents the model (wrapping the anywidget model object), that can have multiple views. + * An object that represents the model(wrapping the anywidget model object), that can have multiple views. */ -class RendercanvasAnywidgetModel { - constructor (anymodel) { +class RenderviewAnywidgetModel { + constructor(anymodel) { this.anymodel = anymodel this.views = [] this._hasVisibleViews = false @@ -46,6 +53,18 @@ class RendercanvasAnywidgetModel { view.setResizable(resizable) } }) + anymodel.on(`change:${prefix}is_minimizable`, () => { + const minimizable = anymodel.get(`${prefix}is_minimizable`) + for (const view of this.views) { + view.setMinimizable(minimizable) + } + }) + anymodel.on(`change:${prefix}is_closable`, () => { + const closable = anymodel.get(`${prefix}is_closable`) + for (const view of this.views) { + view.setClosable(closable) + } + }) anymodel.on(`change:${prefix}has_titlebar`, () => { const titlebar = anymodel.get(`${prefix}has_titlebar`) for (const view of this.views) { @@ -71,7 +90,7 @@ class RendercanvasAnywidgetModel { this._request_animation_frame() } - close () { + close() { URL.revokeObjectURL(this._lastSrc) this._lastSrc = null this._lastFrame = null @@ -88,7 +107,7 @@ class RendercanvasAnywidgetModel { this.onEvent(event) } - addView (view) { + addView(view) { this.views.push(view) this.updateVisibility() // Init attrs @@ -105,12 +124,12 @@ class RendercanvasAnywidgetModel { } } - removeView (view) { + removeView(view) { this.views = this.views.filter(v => v !== view) this.updateVisibility() } - updateVisibility () { + updateVisibility() { let visibleViewsCount = 0 for (const view of this.views) { if (view.isVisible) { visibleViewsCount += 1 } @@ -123,7 +142,7 @@ class RendercanvasAnywidgetModel { } } - _send_response () { + _send_response() { // Let Python know what we have at the frame. This prop is a dict, making it "atomic". const frame = this._lastFrame const frameFeedback = { index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } @@ -131,7 +150,7 @@ class RendercanvasAnywidgetModel { this.anymodel.save_changes() } - _request_animation_frame () { + _request_animation_frame() { // Request an animation frame. // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. @@ -142,7 +161,7 @@ class RendercanvasAnywidgetModel { } } - _animate () { + _animate() { this._img_update_pending = false if (this._frames.length === 0) { return }; @@ -171,7 +190,7 @@ class RendercanvasAnywidgetModel { this._send_response() } - onEvent (event) { + onEvent(event) { try { this.anymodel.send(event) } catch { } // probably attempt to send when widget is closed @@ -182,7 +201,7 @@ class RendercanvasAnywidgetModel { * View to show the anywidget output and observe events, based on renderview.js. */ class AnywidgetRenderView extends BaseRenderView { - constructor (model, containerElement) { + constructor(model, containerElement) { // Create the wrapper element const wrapperElement = document.createElement('div') wrapperElement.classList.add('renderview-wrapper') @@ -206,12 +225,12 @@ class AnywidgetRenderView extends BaseRenderView { this.model.addView(this) } - close () { + close() { super.close() this.model.removeView(this) } - onEvent (event) { + onEvent(event) { if (event.type === 'resize') { // Note that there can be multiple views that can possibly be individually resized. // TODO: keep logical size in check between different views? @@ -236,12 +255,12 @@ class AnywidgetRenderView extends BaseRenderView { export default () => { let model return { - initialize (ctx) { - model = new RendercanvasAnywidgetModel(ctx.model) + initialize(ctx) { + model = new RenderviewAnywidgetModel(ctx.model) // window.model = model // debug return () => { model.close() } }, - render (ctx) { + render(ctx) { const view = new AnywidgetRenderView(model, ctx.el) return () => { view.close() } } From cb3fe2bae51e301e2d96849f13acee9bcbcbad8e Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 12:42:27 +0200 Subject: [PATCH 07/15] Add support for snapshots --- rendercanvas/anywidget.py | 60 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/rendercanvas/anywidget.py b/rendercanvas/anywidget.py index affa92d3..5a65fe63 100644 --- a/rendercanvas/anywidget.py +++ b/rendercanvas/anywidget.py @@ -15,12 +15,13 @@ import numpy as np import anywidget +from IPython.display import display, HTML from traitlets import Bool, Dict, Int, Unicode def _load_js_and_css(): js = "" - for fname in ["renderview.js", "renderview-rfb.js"]: + for fname in ["renderview.js", "renderview-afm.js"]: js_path = resource_files("rendercanvas.core").joinpath(fname) js += js_path.read_text() + "\n\n" @@ -77,6 +78,10 @@ def __init__(self, *args, **kwargs): self._rfb_last_confirmed_index = 0 self._rfb_warned_png = False self._rfb_lossless_draw_info = None + self._rfb_last_resize_event = None + self._rfb_pending_snapshot_display = None + + self._last_set_logical_size = 100, 100 self._use_websocket = True # Could be a prop, private for now self.reset_stats() @@ -89,6 +94,52 @@ def __init__(self, *args, **kwargs): # Set size, title, etc. self._final_canvas_init() + def snapshot(self, *, pixel_ratio=None): + """Render a frame and include the resulting image in the output. + + An initial placeholder output is produced, which is replaced by an html + ```` as soon as the next frame is rendered. + + If the widget is not displayed yet, a resize event is emitted to mimic a widget + size. The ``pixel_ratio`` argument is then used to calculate the physical size. + """ + if self._rfb_last_resize_event is None: + w, h = self._last_set_logical_size + r = float(pixel_ratio) if pixel_ratio is not None else 1.0 + pw, ph = int(w * r), (h * r) + event = { + "type": "resize", + "width": pw / r, + "height": ph / r, + "pwidth": pw, + "pheight": ph, + "ratio": r, + "timestamp": 0, + } + self._rfb_handle_msg(self, event, []) + + self._rfb_pending_snapshot_display = display( + HTML( + "
pending screenshot ..." + ), + display_id=True, + ) + self.request_draw() + + def _replace_snapshot(self, array): + pending_display = self._rfb_pending_snapshot_display + self._rfb_pending_snapshot_display = None + + event = self._rfb_last_resize_event or {} + w = event.get("width", array.shape[1]) + h = event.get("height", array.shape[0]) + + mimetype, data = encode_array(array, 70) + src = f"data:image/{mimetype};base64," + encodebytes(data).decode() + html = f"" + + pending_display.update(HTML(html)) + def _rfb_handle_msg(self, widget, content, buffers): """Receive custom messages and filter our events.""" event_type = content.get("type") @@ -133,7 +184,7 @@ def _rfb_maybe_draw(self): should_draw = ( self._rfb_draw_requested and frames_in_flight < self._max_buffered_frames - and self._has_visible_views + and (self._has_visible_views or self._rfb_pending_snapshot_display) ) # Do the draw if we should. if should_draw: @@ -199,6 +250,10 @@ def _rfb_send_frame(self, array, is_lossless_redraw=False): self._rfb_stats["start_time"] = timestamp self._rfb_last_confirmed_index = self._rfb_frame_index - 1 + # Reload the output if we did not have a frame when the widget was first loaded + if self._rfb_pending_snapshot_display is not None: + self._replace_snapshot(array) + # Compose message and send msg = dict( type="framebufferdata", @@ -303,6 +358,7 @@ def _rc_present_bitmap(self, *, data, format, **kwargs): self._rfb_send_frame(np.asarray(data)) def _rc_set_logical_size(self, width, height): + self._last_set_logical_size = width, height self._css_width = f"{width}px" self._css_height = f"{height}px" From 648b82f679fb7ebc15ba5d6929da444673fcd7a4 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 12:42:51 +0200 Subject: [PATCH 08/15] Add support for minimizing and closing --- examples/rendercanvas.ipynb | 32 +++++++++++++++++++++++--------- rendercanvas/anywidget.py | 18 ++++++++++++++---- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/examples/rendercanvas.ipynb b/examples/rendercanvas.ipynb index 1a02353d..92bc470f 100644 --- a/examples/rendercanvas.ipynb +++ b/examples/rendercanvas.ipynb @@ -43,7 +43,29 @@ "outputs": [], "source": [ "# Set title to non-empty string to show the title bar\n", - "canvas.set_title(\"Rotating cube\")" + "canvas.set_title(\"Rotating cube\")\n", + "canvas.set_minimizable(True)\n", + "canvas.set_closable(True)" + ] + }, + { + "cell_type": "markdown", + "id": "da1ea8e0-e7bd-41a1-b2f5-0084d0d48888", + "metadata": {}, + "source": [ + "## Snapshots\n", + "\n", + "It is also possible to create snapshots. This is also possible without displaying the interactive canvas itself. The benefit of snapshots is that the output remains visible when the notebook is shown in a static viewer. This makes them useful for e.g. tutorials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16a4a4fe-4c64-4c07-8da9-577a1a010fbe", + "metadata": {}, + "outputs": [], + "source": [ + "canvas.snapshot()" ] }, { @@ -74,14 +96,6 @@ "\n", "out" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e8c17002-0ffa-4f11-8bd3-35a38dc5adb0", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/rendercanvas/anywidget.py b/rendercanvas/anywidget.py index 5a65fe63..bee0bb08 100644 --- a/rendercanvas/anywidget.py +++ b/rendercanvas/anywidget.py @@ -58,6 +58,8 @@ class AnywidgetRenderCanvas(BaseRenderCanvas, anywidget.AnyWidget): _css_width = Unicode("500px").tag(sync=True) _css_height = Unicode("300px").tag(sync=True) _resizable = Bool(True).tag(sync=True) + _is_minimizable = Bool(False).tag(sync=True) + _is_closable = Bool(False).tag(sync=True) _has_titlebar = Bool(False).tag(sync=True) _title = Unicode("").tag(sync=True) _cursor = Unicode("default").tag(sync=True) @@ -386,12 +388,20 @@ def set_css_height(self, css_height: str): self._css_height = css_height def set_resizable(self, resizable: bool): - """Set whether the canvas is manually resizable. + """Set whether the canvas is manually resizable.""" + self._resizable = resizable + + def set_minimizable(self, minimizable: bool): + """Set whether the canvas is manually minimizable via a button in the titlebar. - Note that the canvas can only be made resizable if it was attached to a - wrapper HTML element (not directly to a ````). + If all views of the canvas are hidden (out of view or minimized), its rendering is paused, + saving CPU cycles and battery. """ - self._resizable = resizable + self._is_minimizable = minimizable + + def set_closable(self, closable: bool): + """Set whether the canvas is manually closable via a button in the titlebar.""" + self._is_closable = closable # Make available under a common name From 92f32f7958730775795cd57dd0a9d70d2bc4ffd1 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 12:56:51 +0200 Subject: [PATCH 09/15] standrad --- rendercanvas/core/renderview-afm.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/rendercanvas/core/renderview-afm.js b/rendercanvas/core/renderview-afm.js index 658b98f6..79bc4646 100644 --- a/rendercanvas/core/renderview-afm.js +++ b/rendercanvas/core/renderview-afm.js @@ -11,7 +11,7 @@ * An object that represents the model(wrapping the anywidget model object), that can have multiple views. */ class RenderviewAnywidgetModel { - constructor(anymodel) { + constructor (anymodel) { this.anymodel = anymodel this.views = [] this._hasVisibleViews = false @@ -90,7 +90,7 @@ class RenderviewAnywidgetModel { this._request_animation_frame() } - close() { + close () { URL.revokeObjectURL(this._lastSrc) this._lastSrc = null this._lastFrame = null @@ -107,7 +107,7 @@ class RenderviewAnywidgetModel { this.onEvent(event) } - addView(view) { + addView (view) { this.views.push(view) this.updateVisibility() // Init attrs @@ -124,12 +124,12 @@ class RenderviewAnywidgetModel { } } - removeView(view) { + removeView (view) { this.views = this.views.filter(v => v !== view) this.updateVisibility() } - updateVisibility() { + updateVisibility () { let visibleViewsCount = 0 for (const view of this.views) { if (view.isVisible) { visibleViewsCount += 1 } @@ -142,7 +142,7 @@ class RenderviewAnywidgetModel { } } - _send_response() { + _send_response () { // Let Python know what we have at the frame. This prop is a dict, making it "atomic". const frame = this._lastFrame const frameFeedback = { index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } @@ -150,7 +150,7 @@ class RenderviewAnywidgetModel { this.anymodel.save_changes() } - _request_animation_frame() { + _request_animation_frame () { // Request an animation frame. // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. @@ -161,7 +161,7 @@ class RenderviewAnywidgetModel { } } - _animate() { + _animate () { this._img_update_pending = false if (this._frames.length === 0) { return }; @@ -190,7 +190,7 @@ class RenderviewAnywidgetModel { this._send_response() } - onEvent(event) { + onEvent (event) { try { this.anymodel.send(event) } catch { } // probably attempt to send when widget is closed @@ -201,7 +201,7 @@ class RenderviewAnywidgetModel { * View to show the anywidget output and observe events, based on renderview.js. */ class AnywidgetRenderView extends BaseRenderView { - constructor(model, containerElement) { + constructor (model, containerElement) { // Create the wrapper element const wrapperElement = document.createElement('div') wrapperElement.classList.add('renderview-wrapper') @@ -225,12 +225,12 @@ class AnywidgetRenderView extends BaseRenderView { this.model.addView(this) } - close() { + close () { super.close() this.model.removeView(this) } - onEvent(event) { + onEvent (event) { if (event.type === 'resize') { // Note that there can be multiple views that can possibly be individually resized. // TODO: keep logical size in check between different views? @@ -255,12 +255,12 @@ class AnywidgetRenderView extends BaseRenderView { export default () => { let model return { - initialize(ctx) { + initialize (ctx) { model = new RenderviewAnywidgetModel(ctx.model) // window.model = model // debug return () => { model.close() } }, - render(ctx) { + render (ctx) { const view = new AnywidgetRenderView(model, ctx.el) return () => { view.close() } } From e0f7707d810b73ae74dc2cebd13a132329d1c560 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 13:08:31 +0200 Subject: [PATCH 10/15] docs --- docs/backends.rst | 51 ++++++++++++++++++++------------------------ rendercanvas/auto.py | 2 +- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/docs/backends.rst b/docs/backends.rst index 0a9ba88a..ba83d02a 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -20,21 +20,6 @@ The table below gives an overview of the names in the different ``rendercanvas`` | ``RenderCanvas`` (alias) | ``loop`` (an ``AsyncioLoop``) - | A lightweight backend. - * - ``anywidget`` - - | ``AnywidgetRenderCanvas`` - | ``RenderCanvas`` (alias) - | ``loop`` (an ``AsyncioLoop``) - - | Integrate in notebooks. - * - ``jupyter`` - - | ``JupyterRenderCanvas`` - | ``RenderCanvas`` (alias) - | ``loop`` (an ``AsyncioLoop``) - - | Integrate in notebooks via ``jupyter_rfb``. - * - ``offscreen`` - - | ``OffscreenRenderCanvas`` - | ``RenderCanvas`` (alias) - | ``loop`` (a ``StubLoop``) - - | For offscreen rendering. * - ``qt`` - | ``QRenderCanvas`` (toplevel) | ``RenderCanvas`` (alias) @@ -51,6 +36,21 @@ The table below gives an overview of the names in the different ``rendercanvas`` | ``loop`` - | Create a standalone canvas using wx, or | integrate a render canvas in a wx application. + * - ``offscreen`` + - | ``OffscreenRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (a ``StubLoop``) + - | For offscreen rendering. + * - ``anywidget`` + - | ``AnywidgetRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (an ``AsyncioLoop``) + - | Integrate in notebooks using anywidget. + * - ``jupyter`` + - | ``JupyterRenderCanvas`` + | ``RenderCanvas`` (alias) + | ``loop`` (an ``AsyncioLoop``) + - | Integrate in notebooks via ``jupyter_rfb`` (deprecated). * - ``pyodide`` - | ``PyodideRenderCanvas`` (toplevel) | ``RenderCanvas`` (alias) @@ -58,7 +58,6 @@ The table below gives an overview of the names in the different ``rendercanvas`` - | Backend when Python is running in the browser, | via Pyodide or PyScript. - There are also three loop-backends. These are mainly intended for use with the glfw backend: .. list-table:: @@ -270,21 +269,13 @@ object, but in some cases it's convenient to do so with a canvas-like API. Support for notebooks --------------------- -RenderCanvas can be used in Jupyter lab, Jupyter notebook, VSCode, Google Colab, Marimo notebooks, and anywhere else where ``anywidget`` is supported. - -There are two backends that support the notebook: - -* The ``anywidget`` backend. -* The ``jupyter`` backend, which relies on the `jupyter_rfb `_ library. +With the ``anywidget`` backend, RenderCanvas can be used in Jupyter lab, Jupyter notebook, VSCode, Google Colab, Marimo notebooks, and anywhere else where ``anywidget`` is supported. +When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected automatically. -Although they share the most part of their code, the latter has some additional functionality, such as a snapshot utility. -When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected. +The ``jupyter`` backend is the previous backend to provide notebook support, which is based on ``jupyter_rfb``. It's kept for backwards compatibility. .. code-block:: py - - # from rendercanvas.anywidget import RenderCanvas - # from rendercanvas.jupyter import RenderCanvas - from rendercanvas.auto import RenderCanvas # defaults to anywidget when in a notebook + from rendercanvas.auto import RenderCanvas # uses anywidget when in a notebook canvas = RenderCanvas() @@ -293,6 +284,10 @@ When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected. canvas # Use as cell output +.. autoclass:: rendercanvas.anywidget.AnywidgetRenderCanvas + :members: + + Support for Pyodide ------------------- diff --git a/rendercanvas/auto.py b/rendercanvas/auto.py index 47fe4269..04411efe 100644 --- a/rendercanvas/auto.py +++ b/rendercanvas/auto.py @@ -132,7 +132,7 @@ def get_env_var(*varnames): def backends_by_notebook(): - """Generate backend names that are appropriate for the current Jupyter session (if any).""" + """Generate backend names that are appropriate for the current notebook session (if any).""" # Detect Marimo: https://github.com/marimo-team/marimo/discussions/8865 if "marimo" in sys.modules and sys.modules["marimo"].running_in_notebook(): From f1f0b151d18667341d3a8061a4679584791723c9 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 15:36:37 +0200 Subject: [PATCH 11/15] add encoders --- rendercanvas/core/encoders.py | 139 +++++++++++++++++++++++++++++++++ tests/test_encoders.py | 142 ++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 rendercanvas/core/encoders.py create mode 100644 tests/test_encoders.py diff --git a/rendercanvas/core/encoders.py b/rendercanvas/core/encoders.py new file mode 100644 index 00000000..fbca32c5 --- /dev/null +++ b/rendercanvas/core/encoders.py @@ -0,0 +1,139 @@ +import io +import struct +import zlib + +import numpy as np + +try: + import simplejpeg +except ImportError: + simplejpeg = None + + +CAN_JPEG = simplejpeg is not None + + +def encode_array(array, quality: int = 75): + """Encode an image array to a compressed format. + + If the quality is 100, a PNG is returned. Otherwise, JPEG is + preferred and PNG is used as a fallback. Returns (mimetype, bytes). + """ + + if quality >= 100 or not CAN_JPEG: + # Drop alpha channel if it has one + if array.ndim == 3 and array.shape[2] == 4: + array = array[:, :, :3] + mimetype = "image/png" + data = encode_png(array) + else: + mimetype = "image/jpeg" + data = encode_jpeg(array, quality) + + return mimetype, data + + +def encode_jpeg(array, quality: int = 75): + """Encode an image array to bytes to the jpeg format. + + The image shape must be NxM, NxMx3, or NxMx4. + """ + + if simplejpeg is None: + raise RuntimeError("encode_jpeg() needs simplejpeg but it is not installed.") + if not (isinstance(array, np.ndarray) and array.dtype == "uint8"): + raise TypeError("encode_jpeg() requires an uint8 numpy array") + + # Fix and check shape and contiguity + if array.ndim == 2: + array = array.reshape(*array.shape, 1) + if array.ndim == 3 and array.shape[2] == 1: + colorspace = "GRAY" + colorsubsampling = "Gray" + elif array.ndim == 3 and array.shape[2] in (3, 4): + colorspace = "RGBA"[: array.shape[2]] + colorsubsampling = "444" # TODO: does 420 make it faster? + else: + raise ValueError( + f"encode_jpeg() expects an NxM, NxMx3, or NxMx4 array, but got {array.shape}" + ) + + # Make sure it is contiguous + array = np.ascontiguousarray(array) + + # Encode! + return simplejpeg.encode_jpeg( + array, + quality=quality, + colorspace=colorspace, + colorsubsampling=colorsubsampling, + fastdct=True, + ) + + +# TODO: unit tests +def encode_png(array, level: int = 6): + """Encode an image array to bytes to the png format. + + The image shape must be NxM, NxMx3, or NxMx4. + The written image is in RGB or RGBA format, with 8 bit precision, + zlib-compressed, without interlacing. + """ + + if not (isinstance(array, np.ndarray) and array.dtype == "uint8"): + raise TypeError("encode_png() requires an uint8 numpy array") + + # Fix and check shape and contiguity + if array.ndim == 3 and array.shape[2] == 1: + array = array.reshape(*array.shape[:2]) + if array.ndim == 2: + array3 = np.empty((*array.shape[:2], 3), np.uint8) + array3[..., 0] = array + array3[..., 1] = array + array3[..., 2] = array + array = array3 + elif array.ndim == 3 and array.shape[2] in (3, 4): + pass + else: + raise ValueError( + f"encode_png() expects an NxM, NxMx3, or NxMx4 array, but got {array.shape}" + ) + + # Get file object + f = io.BytesIO() + + def add_chunk(data, name): + name = name.encode("ASCII") + crc = zlib.crc32(data, zlib.crc32(name)) + f.write(struct.pack(">I", len(data))) + f.write(name) + f.write(data) + f.write(struct.pack(">I", crc & 0xFFFFFFFF)) + + # Write ... + + # Header + f.write(b"\x89PNG\x0d\x0a\x1a\x0a") + + # First chunk + h, w, c = array.shape + depth = 8 + ctyp = 0b0110 if c == 4 else 0b0010 + ihdr = struct.pack(">IIBBBBB", w, h, depth, ctyp, 0, 0, 0) + add_chunk(ihdr, "IHDR") + + # Chunk with pixels. Just one chunk, no fancy filters. + compressor = zlib.compressobj(level=level) + compressed_data = [] + for row_index in range(array.shape[0]): + row = np.ascontiguousarray(array[row_index]) + compressed_data.append(compressor.compress(b"\x00")) # prepend filter bytes + compressed_data.append(compressor.compress(row)) + compressed_data.append(compressor.flush()) + add_chunk(b"".join(compressed_data), "IDAT") + + # Closing chunk + add_chunk(b"", "IEND") + + f.flush() + return f.getvalue() diff --git a/tests/test_encoders.py b/tests/test_encoders.py new file mode 100644 index 00000000..7e80a431 --- /dev/null +++ b/tests/test_encoders.py @@ -0,0 +1,142 @@ +"""Test the jpeg and png encoders for the remote backends.""" + +from rendercanvas.core import encoders +from rendercanvas.core.encoders import encode_array, encode_jpeg, encode_png +from testutils import run_tests +import pytest +import numpy as np + + +def get_random_im(*shape): + """Get a random image.""" + return np.random.randint(0, 100, shape).astype(np.uint8) + + +def test_encode_array(): + """Test the encode_array function.""" + + # This test assumes that simplejpeg is installed + + im = np.random.randint(0, 255, (100, 100, 3)).astype(np.uint8) + + # Basic check + preamble, bb = encode_array(im) + assert isinstance(preamble, str) + assert isinstance(bb, bytes) + assert "jpeg" in preamble and "png" not in preamble + + # Check compression + preamble1, bb1 = encode_array(im, 90) + preamble2, bb2 = encode_array(im, 30) + assert len(bb2) < len(bb1) + + # Check quality 100 + preamble3, bb3 = encode_array(im, 100) + assert len(bb3) > len(bb1) + + assert "jpeg" in preamble1 and "png" not in preamble1 + assert "jpeg" in preamble2 and "png" not in preamble1 + assert "png" in preamble3 and "jpeg" not in preamble3 + + # Check that RGBA is made RGB + im4 = np.random.randint(0, 255, (100, 100, 4)).astype(np.uint8) + im3 = im4[:, :, :3] + _, bb1 = encode_array(im4, 90) + _, bb2 = encode_array(im3, 90) + assert bb1 == bb2 + + # Also for PNG mode + _, bb1 = encode_array(im4, 100) + _, bb2 = encode_array(im3, 100) + assert bb1 == bb2 + + # Check fallback - disable JPEG encoding, we get PNG + encoders.CAN_JPEG = False + try: + preamble, bb = encode_array(im) + assert isinstance(preamble, str) + assert isinstance(bb, bytes) + assert "png" in preamble and "jpeg" not in preamble + + finally: + encoders.CAN_JPEG = True + + # Should be back to normal now + preamble, bb = encode_array(im) + assert "jpeg" in preamble and "png" not in preamble + + +def test_encode_jpeg(): + """Tests for encode_jpeg function.""" + + _perform_checks(encode_jpeg, 90, 20) + _perform_error_checks(encode_jpeg) + + +def test_encode_png(): + """Tests for encode_jpeg function.""" + + _perform_checks(encode_png, 3, 9) + _perform_error_checks(encode_png) + + +def _perform_checks(encode, c1, c2): + + # Works without compression/level param + im = get_random_im(100, 100, 3) + _bb0 = encode(im) + + # RGB + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # RGB non-contiguous + im = get_random_im(100, 100, 3) + bb1 = encode(im[20:-20, 20:-20, :], c1) + bb2 = encode(im[20:-20, 20:-20, :], c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray1 + im = get_random_im(100, 100) + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray2 + im = get_random_im(100, 100, 1) + bb1 = encode(im, c1) + bb2 = encode(im, c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + # Gray non-contiguous + im = get_random_im(200, 200) + bb1 = encode(im[20:-20, 20:-20], c1) + bb2 = encode(im[20:-20, 20:-20], c2) + assert isinstance(bb1, bytes) + assert len(bb2) < len(bb1) + + +def _perform_error_checks(encode): + # Just to verify that this is ok + encode(get_random_im(10, 10, 3)) + + with pytest.raises(TypeError): # not a numpy array + encode([1, 2, 3, 4]) + + with pytest.raises(TypeError): # not a numpy array + encode(b"1234") + + with pytest.raises(ValueError): # NxMx2? + encode(get_random_im(10, 10, 2)) + + with pytest.raises(TypeError): + encode(get_random_im(10, 10, 3).astype(np.float32)) + + +if __name__ == "__main__": + run_tests(globals()) From fd37fc7b6122cbb5063cdb8de83fbd5fd4014f4b Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 15:39:18 +0200 Subject: [PATCH 12/15] Requiere simplejpeg where it makes sense --- pyproject.toml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9fce626a..5ee86177 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,21 +20,10 @@ keywords = [ requires-python = ">= 3.10" dependencies = ['numpy'] # Numpy is the only hard dependency [project.optional-dependencies] -# For users -jupyter = ["jupyter_rfb>=0.4.2"] +notebook = ["simplejpeg; implementation_name != 'pypy'] glfw = ["glfw>=1.9"] # For devs / ci -lint = ["ruff", "pre-commit"] -examples = ["flit", "numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"] -docs = [ - "flit", - "sphinx>7.2", - "sphinx_rtd_theme", - "sphinx-gallery", - "numpy", - "wgpu", -] -tests = ["pytest", "numpy", "wgpu", "glfw", "trio"] +tests = ["pytest", "numpy", "wgpu", "glfw", "trio", "simplejpeg; implementation_name != 'pypy'"] dev = ["rendercanvas[lint,tests,examples,docs]"] [project.entry-points."pyinstaller40"] @@ -65,4 +54,13 @@ ignore = [ "E731", # Do not assign a `lambda` expression, use a `def` "B007", # Loop control variable `x` not used within loop body "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` -] +]# For userslint = ["ruff", "pre-commit"] +examples = ["flit", "numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"] +docs = [ + "flit", + "sphinx>7.2", + "sphinx_rtd_theme", + "sphinx-gallery", + "numpy", + "wgpu", +]jupyter = ["jupyter_rfb>=0.4.2", "simplejpeg; implementation_name != 'pypy'"] From 4471d2cf6b511dadeba063412ada8f57d76b3ee4 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 15:45:10 +0200 Subject: [PATCH 13/15] fix --- pyproject.toml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ee86177..b18d5697 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,12 +18,24 @@ keywords = [ "jupyter", ] requires-python = ">= 3.10" -dependencies = ['numpy'] # Numpy is the only hard dependency +dependencies = ["numpy"] # Numpy is the only hard dependency + [project.optional-dependencies] -notebook = ["simplejpeg; implementation_name != 'pypy'] +# For users +jupyter = ["jupyter_rfb>=0.4.2", "simplejpeg; implementation_name != 'pypy'"] +notebook = ["simplejpeg; implementation_name != 'pypy'"] glfw = ["glfw>=1.9"] # For devs / ci -tests = ["pytest", "numpy", "wgpu", "glfw", "trio", "simplejpeg; implementation_name != 'pypy'"] +lint = ["ruff", "pre-commit"] +examples = ["flit", "numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"] +docs = ["flit", "sphinx>7.2", "sphinx_rtd_theme", "sphinx-gallery", "wgpu"] +tests = [ + "pytest", + "wgpu", + "glfw", + "trio", + "simplejpeg; implementation_name != 'pypy'", +] dev = ["rendercanvas[lint,tests,examples,docs]"] [project.entry-points."pyinstaller40"] @@ -54,13 +66,4 @@ ignore = [ "E731", # Do not assign a `lambda` expression, use a `def` "B007", # Loop control variable `x` not used within loop body "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` -]# For userslint = ["ruff", "pre-commit"] -examples = ["flit", "numpy", "wgpu", "glfw", "pyside6", "imageio", "pytest"] -docs = [ - "flit", - "sphinx>7.2", - "sphinx_rtd_theme", - "sphinx-gallery", - "numpy", - "wgpu", -]jupyter = ["jupyter_rfb>=0.4.2", "simplejpeg; implementation_name != 'pypy'"] +] From 002f41c6b57d4ac6c7d1e9493f7e4e9a07d429cc Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 15:49:51 +0200 Subject: [PATCH 14/15] fix todos --- rendercanvas/core/encoders.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rendercanvas/core/encoders.py b/rendercanvas/core/encoders.py index fbca32c5..648ed8b4 100644 --- a/rendercanvas/core/encoders.py +++ b/rendercanvas/core/encoders.py @@ -52,7 +52,7 @@ def encode_jpeg(array, quality: int = 75): colorsubsampling = "Gray" elif array.ndim == 3 and array.shape[2] in (3, 4): colorspace = "RGBA"[: array.shape[2]] - colorsubsampling = "444" # TODO: does 420 make it faster? + colorsubsampling = "444" # 420 does not seem to help the compression much else: raise ValueError( f"encode_jpeg() expects an NxM, NxMx3, or NxMx4 array, but got {array.shape}" @@ -71,7 +71,6 @@ def encode_jpeg(array, quality: int = 75): ) -# TODO: unit tests def encode_png(array, level: int = 6): """Encode an image array to bytes to the png format. From 5933789f80b476e0f461a000053ce87b28907c81 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Wed, 8 Apr 2026 16:05:57 +0200 Subject: [PATCH 15/15] docs import anywidget without needing anywidget and friends --- docs/backends.rst | 1 + docs/conf.py | 26 ++++++++++++++++++++++++++ pyproject.toml | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/backends.rst b/docs/backends.rst index ba83d02a..632c5ae3 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -275,6 +275,7 @@ When the ``auto`` backend is used in a notebook, the ``anywidget`` is selected a The ``jupyter`` backend is the previous backend to provide notebook support, which is based on ``jupyter_rfb``. It's kept for backwards compatibility. .. code-block:: py + from rendercanvas.auto import RenderCanvas # uses anywidget when in a notebook canvas = RenderCanvas() diff --git a/docs/conf.py b/docs/conf.py index 00bca06c..27d3a553 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,6 +22,32 @@ os.environ["RENDERCANVAS_FORCE_OFFSCREEN"] = "true" +class FakeTrait: + def __init__(self, *args, **kwargs): + pass + + def tag(self, *args, **kwargs): + pass + + +class FakeModule: + # from anywidget import AnyWidget + AnyWidget = object + # from traitlets import ... + Dict = FakeTrait + Unicode = FakeTrait + Int = FakeTrait + Bool = FakeTrait + # from IPython.display import ... + HTML = lambda *a, **kw: None + display = lambda *a, **kw: None + + +sys.modules["anywidget"] = FakeModule +sys.modules["traitlets"] = FakeModule +sys.modules["IPython.display"] = FakeModule + + # Load wgpu so autodoc can query docstrings import rendercanvas # noqa: E402 import rendercanvas.stub # noqa: E402 - we use the stub backend to generate docs diff --git a/pyproject.toml b/pyproject.toml index b18d5697..166070db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = ["numpy"] # Numpy is the only hard dependency [project.optional-dependencies] # For users jupyter = ["jupyter_rfb>=0.4.2", "simplejpeg; implementation_name != 'pypy'"] -notebook = ["simplejpeg; implementation_name != 'pypy'"] +notebook = ["anywidget", "simplejpeg; implementation_name != 'pypy'"] glfw = ["glfw>=1.9"] # For devs / ci lint = ["ruff", "pre-commit"]