From bfe79a738ec86635572343a436c2f2dba91b4449 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Thu, 26 Mar 2026 14:54:18 +0100 Subject: [PATCH 01/12] http backend --- rendercanvas/http.py | 222 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 rendercanvas/http.py diff --git a/rendercanvas/http.py b/rendercanvas/http.py new file mode 100644 index 00000000..e3f9f747 --- /dev/null +++ b/rendercanvas/http.py @@ -0,0 +1,222 @@ +""" +A remote backend one or more browser views. + +This module implements an ASGI web application, so it runs on any ASGI server. We default to uvicorn. +""" + +import json +import asyncio +from importlib.resources import files as resource_files + +from .asyncio import AsyncioLoop + + +HTML = """ + + + Minimal WS App + + + + +

WebSocket Test

+ +
+ Loading ... +
+ + + +

+
+  
+  
+
+
+"""
+
+
+def _load_resource(fname):
+    return resource_files("rendercanvas.core").joinpath(fname).read_text()
+
+
+# A dict with resources to serve. It maps path -> (content-type, body)
+resources = {}
+resources["/"] = "text/html", HTML
+resources["/index.html"] = "text/html", HTML
+resources["/renderview.js"] = "text/javascript", _load_resource("renderview.js")
+resources["/renderview.css"] = "text/css", _load_resource("renderview.css")
+
+
+class Websocket:
+    """An ASGI websocket
+
+    Each websocket represents one view. These could be in the same browser
+    window, or in different continents.
+    """
+
+    def __init__(self, app, id):
+        self._app = app
+        self._id = id
+        self._send_queue = asyncio.Queue()
+
+    async def _websocket_receiver(self, receive):
+        while True:
+            event = await receive()
+            if event["type"] == "websocket.receive":
+                if "text" in event:
+                    self._on_receive(event["text"])
+                elif "bytes" in event:
+                    self._on_receive(event["bytes"])
+            elif event["type"] == "websocket.disconnect":
+                break
+
+    async def _websocket_sender(self, send):
+        while True:
+            msg = await self._send_queue.get()
+            if isinstance(msg, str):
+                await send({"type": "websocket.send", "text": msg})
+            else:
+                await send({"type": "websocket.send", "bytes": msg})
+
+    def send(self, data):
+        """Send data into the websocket."""
+        _ = self._send_queue.put(data)
+
+    def _on_receive(self, msg):
+        if isinstance(msg, bytes):
+            print("Unexpectedly received bytes ({len(msg}).")
+        try:
+            event = json.loads(msg)
+        except Exception:
+            short_text = text[:100] + "…" if len(text) > 100 else text
+            print(f"Received non-json message: {short_text!r}")
+            return
+        else:
+            # todo: some messages, like frame feedback, should be processed per-ws, others only by one.
+            self._app._on_event(self._id, event)
+
+
+# TODO: how does this work when ppl want to include this in a larger web application, with e.g. FastAPI or Falcon?
+
+
+class Asgi:
+    """The ASGI application.
+
+    This is pretty low-level web-server code, but it means we have minimal dependencies.
+
+    One server, one canvas. So can create only one canvas in a process. Unless
+    we can have multiple ASGI apps running simultaneously, e.g. on different ports or paths.
+
+    One websocket for each client. But only first websocket in the list controls.
+    """
+
+    def __init__(self, resources):
+        self._resources = resources
+        self._websockets = []
+        self._event_callback = lambda ev: None
+        self._ws_count = 0
+
+    async def __call__(self, scope, receive, send):
+        """The ASGI entrypoint."""
+
+        if scope["type"] == "http":
+            content_type_and_body = self._resources.get(scope["path"], None)
+            if content_type_and_body is not None:
+                content_type, body = content_type_and_body
+                if isinstance(body, str):
+                    body = body.encode()
+                await send(
+                    {
+                        "type": "http.response.start",
+                        "status": 200,
+                        "headers": [(b"content-type", content_type.encode())],
+                    }
+                )
+                await send({"type": "http.response.body", "body": body})
+            else:
+                await send(
+                    {
+                        "type": "http.response.start",
+                        "status": 404,
+                        "headers": [(b"content-type", b"text/plain")],
+                    }
+                )
+                await send({"type": "http.response.body", "body": b"Not Found"})
+
+        elif scope["type"] == "websocket":
+            await send({"type": "websocket.accept"})
+
+            self._ws_count += 1
+            ws = Websocket(self, self._ws_count)
+            self._websockets.append(ws)
+
+            try:
+                receiver = asyncio.create_task(ws._websocket_receiver(receive))
+                sender = asyncio.create_task(ws._websocket_sender(send))
+                done, pending = await asyncio.wait(
+                    [receiver, sender],
+                    return_when=asyncio.FIRST_COMPLETED,
+                )
+                for task in pending:
+                    task.cancel()
+            finally:
+                self._websockets.remove(ws)
+
+    def _on_event(self, id, event):
+        """Called when a websocket receives an event."""
+        print(id, event)
+
+    def send_all(self, data: bytes):
+        """Send data to all websockets."""
+        for ws in self._websockets:
+            ws.send(data)
+
+
+class HttpLoop(AsyncioLoop):
+    def run(self, host="localhost", port=60649):
+        self._host = host
+        self._port = port
+        return super().run()
+
+    def _rc_run(self):
+        from uvicorn.main import main as uvicorn_main
+
+        print(f"Starting server at http://{self._host}:{self._port}")
+        uvicorn_main(
+            [
+                f"--host={self._host}",
+                f"--port={self._port}",
+                "--log-level=warning",
+                f"{__name__}:asgi",
+            ]
+        )
+
+
+asgi = Asgi(resources)
+
+loop = HttpLoop()

From f681a6edcf650e8f0562f5551dd6388a51122368 Mon Sep 17 00:00:00 2001
From: Almar Klein 
Date: Fri, 27 Mar 2026 10:15:27 +0100
Subject: [PATCH 02/12] Implement Python side

---
 rendercanvas/http.py | 394 +++++++++++++++++++++++++++++++++++++------
 1 file changed, 344 insertions(+), 50 deletions(-)

diff --git a/rendercanvas/http.py b/rendercanvas/http.py
index e3f9f747..5a474e40 100644
--- a/rendercanvas/http.py
+++ b/rendercanvas/http.py
@@ -4,11 +4,18 @@
 This module implements an ASGI web application, so it runs on any ASGI server. We default to uvicorn.
 """
 
+__all__ = ["HttpRenderCanvas", "RenderCanvas", "asgi", "loop"]
+
 import json
+import time
 import asyncio
 from importlib.resources import files as resource_files
 
+from .base import BaseCanvasGroup, BaseRenderCanvas, logger
 from .asyncio import AsyncioLoop
+from .core.encoders import encode_array, CAN_JPEG
+
+import numpy as np
 
 
 HTML = """
@@ -24,37 +31,35 @@
     
Loading ...
- - - -

-
-  
-  
+    

+ + +

+
+    
+    
 
 
 """
@@ -86,39 +91,48 @@ def __init__(self, app, id):
 
     async def _websocket_receiver(self, receive):
         while True:
-            event = await receive()
+            event = await receive()  # asgi event
             if event["type"] == "websocket.receive":
                 if "text" in event:
                     self._on_receive(event["text"])
                 elif "bytes" in event:
                     self._on_receive(event["bytes"])
             elif event["type"] == "websocket.disconnect":
+                # TODO: handle closing from client side
                 break
 
     async def _websocket_sender(self, send):
         while True:
             msg = await self._send_queue.get()
-            if isinstance(msg, str):
+            if msg is None:
+                await send({"type": "websocket.close", "code": 1000})
+            elif isinstance(msg, str):
                 await send({"type": "websocket.send", "text": msg})
             else:
                 await send({"type": "websocket.send", "bytes": msg})
 
+    def _on_receive(self, text_or_bytes: str | bytes):
+        if isinstance(text_or_bytes, bytes):
+            print("Unexpectedly received bytes ({len(msg}).")
+        else:
+            text = text_or_bytes
+            try:
+                event = json.loads(text)  # JS event
+            except Exception:
+                short_text = text[:100] + "…" if len(text) > 100 else text
+                print(f"Received non-json message: {short_text!r}")
+                return
+            else:
+                # todo: some messages, like frame feedback, should be processed per-ws, others only by one.
+                self._app._on_event(self._id, event)
+
     def send(self, data):
         """Send data into the websocket."""
         _ = self._send_queue.put(data)
 
-    def _on_receive(self, msg):
-        if isinstance(msg, bytes):
-            print("Unexpectedly received bytes ({len(msg}).")
-        try:
-            event = json.loads(msg)
-        except Exception:
-            short_text = text[:100] + "…" if len(text) > 100 else text
-            print(f"Received non-json message: {short_text!r}")
-            return
-        else:
-            # todo: some messages, like frame feedback, should be processed per-ws, others only by one.
-            self._app._on_event(self._id, event)
+    def close(self):
+        """Close the websocket from our end."""
+        _ = self._send_queue.put(None)  # None means close, see _websocket_sender()
 
 
 # TODO: how does this work when ppl want to include this in a larger web application, with e.g. FastAPI or Falcon?
@@ -138,7 +152,7 @@ class Asgi:
     def __init__(self, resources):
         self._resources = resources
         self._websockets = []
-        self._event_callback = lambda ev: None
+        self._event_callback = lambda id, ev: None
         self._ws_count = 0
 
     async def __call__(self, scope, receive, send):
@@ -178,7 +192,7 @@ async def __call__(self, scope, receive, send):
             try:
                 receiver = asyncio.create_task(ws._websocket_receiver(receive))
                 sender = asyncio.create_task(ws._websocket_sender(send))
-                done, pending = await asyncio.wait(
+                _done, pending = await asyncio.wait(
                     [receiver, sender],
                     return_when=asyncio.FIRST_COMPLETED,
                 )
@@ -189,13 +203,19 @@ async def __call__(self, scope, receive, send):
 
     def _on_event(self, id, event):
         """Called when a websocket receives an event."""
-        print(id, event)
+        self._event_callback(id, event)
 
     def send_all(self, data: bytes):
         """Send data to all websockets."""
         for ws in self._websockets:
             ws.send(data)
 
+    def close(self):
+        """Disconnect all clients."""
+        # TODO: also put in a closed (non-restartable) state? i.e. think about lifetime cycle
+        for ws in self._websockets:
+            ws.close()
+
 
 class HttpLoop(AsyncioLoop):
     def run(self, host="localhost", port=60649):
@@ -217,6 +237,280 @@ def _rc_run(self):
         )
 
 
-asgi = Asgi(resources)
-
 loop = HttpLoop()
+
+
+class HttpCanvasGroup(BaseCanvasGroup):
+    pass
+
+
+class HttpRenderCanvas(BaseRenderCanvas):
+    """A remote canvas that is served over http and viewed in a browser."""
+
+    _rc_canvas_group = HttpCanvasGroup(loop)
+
+    _max_buffered_frames = 2
+    _quality = 80
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # todo: limit to a single canvas
+        asgi._event_callback = self._on_event
+
+        self._is_closed = False
+
+        self._draw_requested = False
+        self._frame_index = 0
+        self._last_confirmed_index = 0
+        self._warned_png = False
+        self._lossless_draw_info = None
+
+        self.reset_stats()
+
+        # Set size, title, etc.
+        self._final_canvas_init()
+
+    def _on_event(self, id: int, event: dict):
+        try:
+            type = event["type"]
+        except KeyError:
+            logger.warning(f"Invalid event: {event!r}")
+            return
+
+        # TODO: some logic depends on the id
+        # TODO: keep track of frame feedback per id, main ws determines frame rate, others drop frames as necessary
+
+        if type.startswith("comm-"):
+            if type == "comm-frame-feedback":
+                self._frame_feedback = event["value"]
+                loop.call_soon(self._maybe_draw)
+            elif type == "comm-has-visible-views":
+                self._has_visible_views = event["value"]
+                loop.call_soon(self._maybe_draw)
+            else:
+                logger.warning(f"Unknown comm event: {event!r}")
+        else:
+            # A renderview event
+
+            if type == "resize":
+                self._size_info.set_physical_size(
+                    event["pwidth"], event["pheight"], event["ratio"]
+                )
+            elif type == "close":
+                self.close()
+            else:
+                # Compatibility between new renderview event spec and current rendercanvas/pygfx events
+                event["event_type"] = event.pop("type")
+                event["time_stamp"] = event.pop("timestamp")
+                # Turn lists into tuples (js/json does not have tuples)
+                if "buttons" in event:
+                    event["buttons"] = tuple(event["buttons"])
+                if "modifiers" in event:
+                    event["modifiers"] = tuple(event["modifiers"])
+                self.submit_event(event)
+
+    def _maybe_draw(self):
+        """Perform a draw, if we can and should."""
+        feedback = self._frame_feedback
+        # Update stats
+        self._update_stats(feedback)
+        # Determine whether we should perform a draw: a draw was requested, and
+        # the client is ready for a new frame, and the client widget is visible.
+        frames_in_flight = self._frame_index - feedback.get("index", 0)
+        should_draw = (
+            self._draw_requested
+            and frames_in_flight < self._max_buffered_frames
+            and self._has_visible_views
+        )
+        # Do the draw if we should.
+        if should_draw:
+            self._draw_requested = False
+            self._time_to_draw()  # -> _rc_present_bitmap -> _send_frame
+
+    def _schedule_lossless_draw(self, array, delay=0.3):
+        self._cancel_lossless_draw()
+        loop = asyncio.get_running_loop()
+        handle = loop.call_later(delay, self._lossless_draw)
+        self._lossless_draw_info = array, handle
+
+    def _cancel_lossless_draw(self):
+        if self._lossless_draw_info:
+            _, handle = self._lossless_draw_info
+            self._lossless_draw_info = None
+            handle.cancel()
+
+    def _lossless_draw(self):
+        array, _ = self._lossless_draw_info
+        self._send_frame(array, True)
+
+    def _send_frame(self, array, is_lossless_redraw=False):
+        """Actually send a frame over to the client."""
+        # For considerations about performance,
+        # see https://github.com/vispy/jupyter_rfb/issues/3
+
+        quality = 100 if is_lossless_redraw else self._quality
+
+        self._frame_index += 1
+        timestamp = time.time()
+
+        # Turn array into a based64-encoded JPEG or PNG
+        t1 = time.perf_counter()
+        mimetype, data = encode_array(array, quality)
+        datas = [data]
+        data_b64 = None
+        t2 = time.perf_counter()
+
+        if "jpeg" in mimetype:
+            self._schedule_lossless_draw(array)
+        else:
+            self._cancel_lossless_draw()
+            # Issue png warning?
+            if quality < 100 and not CAN_JPEG and not self._warned_png:
+                self._warned_png = True
+                logger.warning(
+                    "No JPEG encoder found, using PNG instead. Install simplejpeg for better performance."
+                )
+
+        if is_lossless_redraw:
+            # No stats, also not on the confirmation of this frame
+            self._last_confirmed_index = self._frame_index
+        else:
+            # Stats
+            self._stats["img_encoding_sum"] += t2 - t1
+            self._stats["sent_frames"] += 1
+            if self._stats["start_time"] <= 0:  # Start measuring
+                self._stats["start_time"] = timestamp
+                self._last_confirmed_index = self._frame_index - 1
+
+        # Compose message and send
+        msg = dict(
+            type="framebufferdata",
+            mimetype=mimetype,
+            data_b64=data_b64,
+            index=self._frame_index,
+            timestamp=timestamp,
+        )
+        self.send(msg, datas)
+
+    # ----- related to stats
+
+    def reset_stats(self):
+        """Restart measuring statistics from the next sent frame."""
+        self._stats = {
+            "start_time": 0,
+            "last_time": 1,
+            "sent_frames": 0,
+            "confirmed_frames": 0,
+            "roundtrip_count": 0,
+            "roundtrip_sum": 0,
+            "delivery_sum": 0,
+            "img_encoding_sum": 0,
+        }
+
+    def get_stats(self):
+        """Get the current stats since the last time ``.reset_stats()`` was called.
+
+        Stats is a dict with the following fields:
+
+        * *sent_frames*: the number of frames sent.
+        * *confirmed_frames*: number of frames confirmed by the client.
+        * *roundtrip*: average time for processing a frame, including receiver confirmation.
+        * *delivery*: average time for processing a frame until it's received by the client.
+          This measure assumes that the clock of the server and client are precisely synced.
+        * *img_encoding*: the average time spent on encoding the array into an image.
+        * *b64_encoding*: the average time spent on base64 encoding the data.
+        * *fps*: the average FPS, measured from the first frame sent since ``.reset_stats()``
+          was called, until the last confirmed frame.
+        """
+        d = self._stats
+        roundtrip_count_div = d["roundtrip_count"] or 1
+        sent_frames_div = d["sent_frames"] or 1
+        fps_div = (d["last_time"] - d["start_time"]) or 0.001
+        return {
+            "sent_frames": d["sent_frames"],
+            "confirmed_frames": d["confirmed_frames"],
+            "roundtrip": d["roundtrip_sum"] / roundtrip_count_div,
+            "delivery": d["delivery_sum"] / roundtrip_count_div,
+            "img_encoding": d["img_encoding_sum"] / sent_frames_div,
+            "fps": d["confirmed_frames"] / fps_div,
+        }
+
+    def _update_stats(self, feedback):
+        """Update the stats when a new frame feedback has arrived."""
+        last_index = feedback.get("index", 0)
+        if last_index > self._last_confirmed_index:
+            timestamp = feedback["timestamp"]
+            nframes = last_index - self._last_confirmed_index
+            self._last_confirmed_index = last_index
+            self._stats["confirmed_frames"] += nframes
+            self._stats["roundtrip_count"] += 1
+            self._stats["roundtrip_sum"] += time.time() - timestamp
+            self._stats["delivery_sum"] += feedback["localtime"] - timestamp
+            self._stats["last_time"] = time.time()
+
+    # --- the API to be a rendercanvas backend
+
+    def _rc_gui_poll(self):
+        pass
+
+    def _rc_get_present_info(self, present_methods):
+        # Only allow simple format for now. srgb is assumed.
+        if "bitmap" in present_methods:
+            return {
+                "method": "bitmap",
+                "formats": ["rgba-u8"],
+            }
+        else:
+            return None  # raises error
+
+    def _rc_request_draw(self):
+        # Technically, _maybe_draw() may not perform a draw if there are too
+        # many frames in-flight. But in this case, we'll eventually get
+        # new frame_feedback, which will then trigger a draw.
+        if not self._draw_requested:
+            self._draw_requested = True
+            self._cancel_lossless_draw()
+            loop.call_soon(self._maybe_draw)
+
+    def _rc_request_paint(self):
+        # We technically don't need to call _time_to_paint, because this backend only does bitmap mode.
+        # But in case the base backend will do something in _time_to_paint later, we behave nice.
+        loop = self._rc_canvas_group.get_loop()
+        loop.call_soon(self._time_to_paint)
+
+    def _rc_force_paint(self):
+        pass  # works as-is via push_frame
+
+    def _rc_present_bitmap(self, *, data, format, **kwargs):
+        assert format == "rgba-u8"
+        self._send_frame(np.asarray(data))
+
+    def _rc_set_logical_size(self, width, height):
+        asgi.send({"type": "comm-css-width", "value": f"{width}px"})
+        asgi.send({"type": "comm-css-height", "value": f"{height}px"})
+
+    def _rc_close(self):
+        asgi.close()
+        self._is_closed = True
+
+    def _rc_get_closed(self):
+        return self._is_closed
+
+    def _rc_set_title(self, title):
+        asgi.send({"type": "comm-title", "value": title})
+
+    def _rc_set_cursor(self, cursor):
+        asgi.send({"type": "comm-cursor", "value": cursor})
+
+    def set_css_width(self, css_width: str):
+        """Set the width of the canvas as a CSS string."""
+        asgi.send({"type": "comm-css-width", "value": css_width})
+
+    def set_css_height(self, css_height: str):
+        """Set the height of the canvas as a CSS string."""
+        asgi.send({"type": "comm-css-height", "value": css_height})
+
+
+asgi = Asgi(resources)
+RenderCanvas = HttpRenderCanvas

From 88b010ff24fbedebd78b43783afb2d367f96d19d Mon Sep 17 00:00:00 2001
From: Almar Klein 
Date: Thu, 9 Apr 2026 15:26:38 +0200
Subject: [PATCH 03/12] Progress

---
 rendercanvas/core/renderview-client.js | 153 +++++++++++++++++++++++++
 rendercanvas/http.py                   | 126 +++++++++++---------
 2 files changed, 227 insertions(+), 52 deletions(-)
 create mode 100644 rendercanvas/core/renderview-client.js

diff --git a/rendercanvas/core/renderview-client.js b/rendercanvas/core/renderview-client.js
new file mode 100644
index 00000000..43f3f2d0
--- /dev/null
+++ b/rendercanvas/core/renderview-client.js
@@ -0,0 +1,153 @@
+/* Code to use renderview in a remote browser (http backend). Some of the logic is very similar to 'renderview-afm.js'. */
+
+/* global BaseRenderView WebSocket */
+
+class ClientRenderView extends BaseRenderView {
+  constructor (wrapperElement) {
+    wrapperElement.classList.add('renderview-wrapper')
+
+    // Create view element
+    const viewElement = document.createElement('img')
+    viewElement.decoding = 'sync'
+    viewElement.loading = 'eager'
+    viewElement.style.touchAction = 'none' // prevent default pan/zoom behavior
+    viewElement.ondragstart = () => false // prevent browser's built-in image drag
+
+    // Instantiate
+    super(viewElement, wrapperElement)
+    this.setThrottle(20) // 20ms -> max 50 move/wheel events per second
+
+    this.frames = []
+    this.imgUpdatePending = false
+    this.lastSrc = null
+  }
+
+  onEvent (event) {
+    if (websocket !== null) {
+      websocket.send(JSON.stringify(event))
+    }
+  }
+
+  requestAnimationFrame () {
+    // Request an animation frame.
+    // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth,
+    // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that.
+    if (!this.imgUpdatePending) {
+      this.imgUpdatePending = true
+      window.requestAnimationFrame(this.animate.bind(this))
+    }
+  }
+
+  animate () {
+    this.imgUpdatePending = false
+    if (this.frames.length === 0) { return };
+
+    // Pick the oldest frame from the stack, and get its source
+    const frame = this.frames.shift()
+    let newSrc
+    if (frame.buffers && frame.buffers.length > 0) {
+      const blob = new Blob([frame.buffers[0].buffer], { type: frame.mimetype })
+      newSrc = URL.createObjectURL(blob)
+    } else {
+      newSrc = frame.data_b64
+    }
+
+    // Revoke last objectURL
+    URL.revokeObjectURL(this.lastSrc)
+    this.lastSrc = newSrc
+
+    // Update the image sources
+    view.viewElement.src = newSrc
+    view.viewElement.onload = this.requestAnimationFrame.bind(this)
+
+    // Let the server know we processed the image (even if it's not shown yet)
+    this.sendResponse(frame)
+  }
+
+  sendResponse (frame) {
+    // Let Python know what we have at the frame.
+    const event = { type: '_framefeedback', index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 }
+    this.onEvent(event)
+  }
+}
+
+const wrapperElement = document.getElementById('canvas')
+const statusElement = document.getElementById('status')
+let view = null
+let websocket = null
+
+updateStatus()
+openWebsocketConnection()
+window.openWebsocketConnection = openWebsocketConnection
+
+function updateStatus () {
+  if (statusElement === null) { return }
+
+  if (websocket === null) {
+    statusElement.innerHTML = "? Disconnected "
+  } else {
+    statusElement.innerHTML = "+ Connected"
+  }
+}
+
+function openWebsocketConnection () {
+  const ws = new WebSocket('ws://' + window.location.host + '/ws')
+
+  ws.onopen = (e) => {
+    console.log('websocket opened')
+    websocket = ws
+    window.websocket = ws // allow manual closing to mimic lost connection
+    if (view === null) {
+      view = new ClientRenderView(wrapperElement)
+      console.log('created ClientRenderView')
+    }
+    updateStatus()
+  }
+  ws.onerror = (e) => {
+    console.log(`websocket error: ${e}`)
+    websocket = null
+    updateStatus()
+  }
+
+  let pendingMsg
+  ws.onmessage = (e) => {
+    let msg = null
+
+    // First some handling to support a message with buffers
+    if (typeof e.data === 'string' || e.data instanceof String) {
+      msg = JSON.parse(e.data)
+      if (msg.nbuffers && msg.nbuffers > 0) {
+        pendingMsg = msg
+        pendingMsg.buffers = []
+        msg = null
+      } else {
+        pendingMsg = null // discard unfinished pending message (if any)
+      }
+    } else { // Blob
+      if (pendingMsg !== null) {
+        pendingMsg.buffers.push(e.data)
+        if (pendingMsg.buffers.length >= pendingMsg.nbuffers) {
+          msg = pendingMsg
+          pendingMsg = null
+        }
+      }
+    }
+
+    if (msg === null) { return }
+
+    // Process message
+    // console.log(msg)
+    if (msg.type === 'framebufferdata') {
+      view.frames.push(msg)
+      view.requestAnimationFrame()
+    } else if (msg.type === 'cursor') {
+      view.setCursor(msg.value)
+    }
+  }
+
+  ws.onclose = (e) => {
+    console.log(`websocket closed: ${e.reason} (${e.code})`)
+    websocket = null
+    updateStatus()
+  }
+}
diff --git a/rendercanvas/http.py b/rendercanvas/http.py
index 5a474e40..0a677219 100644
--- a/rendercanvas/http.py
+++ b/rendercanvas/http.py
@@ -14,6 +14,7 @@
 from .base import BaseCanvasGroup, BaseRenderCanvas, logger
 from .asyncio import AsyncioLoop
 from .core.encoders import encode_array, CAN_JPEG
+from .core.events import valid_event_types
 
 import numpy as np
 
@@ -21,45 +22,23 @@
 HTML = """
 
 
-    Minimal WS App
-    
+    RenderCanvas over http
+    
+    
     
 
 
-  

WebSocket Test

+

RenderCanvas over http

-
- Loading ... +
+

Loading ...




 
-    
-    
+    
""" @@ -73,8 +52,9 @@ def _load_resource(fname): resources = {} resources["/"] = "text/html", HTML resources["/index.html"] = "text/html", HTML -resources["/renderview.js"] = "text/javascript", _load_resource("renderview.js") resources["/renderview.css"] = "text/css", _load_resource("renderview.css") +for fname in ("renderview.js", "renderview-afm.js", "renderview-client.js"): + resources[f"/{fname}"] = "text/javascript", _load_resource(fname) class Websocket: @@ -128,7 +108,13 @@ def _on_receive(self, text_or_bytes: str | bytes): def send(self, data): """Send data into the websocket.""" - _ = self._send_queue.put(data) + if isinstance(data, dict): + data = json.dumps(data) + elif isinstance(data, bytes): + data = data + else: + RuntimeError("ws.send expects dict or bytes") + asyncio.create_task(self._send_queue.put(data)) def close(self): """Close the websocket from our end.""" @@ -158,7 +144,23 @@ def __init__(self, resources): async def __call__(self, scope, receive, send): """The ASGI entrypoint.""" - if scope["type"] == "http": + if scope["type"] == "lifespan": + while True: + message = await receive() + if message["type"] == "lifespan.startup": + try: + asyncio.get_running_loop().create_task(loop._rc_run_async()) + except Exception as err: + print("could not start rendercanvas loop:", err) + else: + print("rendercanvas loop started in server") + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + ... # Do some shutdown here! + await send({"type": "lifespan.shutdown.complete"}) + return + + elif scope["type"] == "http": content_type_and_body = self._resources.get(scope["path"], None) if content_type_and_body is not None: content_type, body = content_type_and_body @@ -188,6 +190,7 @@ async def __call__(self, scope, receive, send): self._ws_count += 1 ws = Websocket(self, self._ws_count) self._websockets.append(ws) + self._event_callback(0, {"type": "_new_client"}) try: receiver = asyncio.create_task(ws._websocket_receiver(receive)) @@ -203,9 +206,12 @@ async def __call__(self, scope, receive, send): def _on_event(self, id, event): """Called when a websocket receives an event.""" - self._event_callback(id, event) + try: + self._event_callback(id, event) + except Exception as err: + print(f"Error handling ws event callback: {err}") - def send_all(self, data: bytes): + def send_all(self, data: str | bytes): """Send data to all websockets.""" for ws in self._websockets: ws.send(data) @@ -216,6 +222,9 @@ def close(self): for ws in self._websockets: ws.close() + def get_count(self): + return len(self._websockets) + class HttpLoop(AsyncioLoop): def run(self, host="localhost", port=60649): @@ -261,6 +270,7 @@ def __init__(self, *args, **kwargs): self._is_closed = False self._draw_requested = False + self._frame_feedback = {} self._frame_index = 0 self._last_confirmed_index = 0 self._warned_png = False @@ -281,12 +291,13 @@ def _on_event(self, id: int, event: dict): # TODO: some logic depends on the id # TODO: keep track of frame feedback per id, main ws determines frame rate, others drop frames as necessary - if type.startswith("comm-"): - if type == "comm-frame-feedback": - self._frame_feedback = event["value"] + if type.startswith("_"): + if type == "_framefeedback": + self._frame_feedback = event loop.call_soon(self._maybe_draw) - elif type == "comm-has-visible-views": - self._has_visible_views = event["value"] + elif type == "_new_client": + # Force a draw. With a new client, we want to override frame feedback + self._draw_requested = 2 loop.call_soon(self._maybe_draw) else: logger.warning(f"Unknown comm event: {event!r}") @@ -299,7 +310,7 @@ def _on_event(self, id: int, event: dict): ) elif type == "close": self.close() - else: + elif type in valid_event_types: # Compatibility between new renderview event spec and current rendercanvas/pygfx events event["event_type"] = event.pop("type") event["time_stamp"] = event.pop("timestamp") @@ -320,8 +331,11 @@ def _maybe_draw(self): frames_in_flight = self._frame_index - feedback.get("index", 0) should_draw = ( self._draw_requested - and frames_in_flight < self._max_buffered_frames - and self._has_visible_views + and ( + frames_in_flight < self._max_buffered_frames + or self._draw_requested >= 2 + ) + and asgi.get_count() > 0 ) # Do the draw if we should. if should_draw: @@ -348,7 +362,6 @@ def _send_frame(self, array, is_lossless_redraw=False): """Actually send a frame over to the client.""" # For considerations about performance, # see https://github.com/vispy/jupyter_rfb/issues/3 - quality = 100 if is_lossless_redraw else self._quality self._frame_index += 1 @@ -357,8 +370,14 @@ def _send_frame(self, array, is_lossless_redraw=False): # Turn array into a based64-encoded JPEG or PNG t1 = time.perf_counter() mimetype, data = encode_array(array, quality) - datas = [data] - data_b64 = None + if False: # use_websocket: + buffers = [data] + data_b64 = None + else: + buffers = [] + from base64 import encodebytes + + data_b64 = f"data:{mimetype};base64," + encodebytes(data).decode() t2 = time.perf_counter() if "jpeg" in mimetype: @@ -386,12 +405,15 @@ def _send_frame(self, array, is_lossless_redraw=False): # Compose message and send msg = dict( type="framebufferdata", + nbuffers=len(buffers), mimetype=mimetype, data_b64=data_b64, index=self._frame_index, timestamp=timestamp, ) - self.send(msg, datas) + asgi.send_all(msg) + for buffer in buffers: + asgi.send_all(buffer) # ----- related to stats @@ -487,8 +509,8 @@ def _rc_present_bitmap(self, *, data, format, **kwargs): self._send_frame(np.asarray(data)) def _rc_set_logical_size(self, width, height): - asgi.send({"type": "comm-css-width", "value": f"{width}px"}) - asgi.send({"type": "comm-css-height", "value": f"{height}px"}) + asgi.send_all({"type": "comm-css-width", "value": f"{width}px"}) + asgi.send_all({"type": "comm-css-height", "value": f"{height}px"}) def _rc_close(self): asgi.close() @@ -498,18 +520,18 @@ def _rc_get_closed(self): return self._is_closed def _rc_set_title(self, title): - asgi.send({"type": "comm-title", "value": title}) + asgi.send_all({"type": "comm-title", "value": title}) def _rc_set_cursor(self, cursor): - asgi.send({"type": "comm-cursor", "value": cursor}) + asgi.send_all({"type": "comm-cursor", "value": cursor}) def set_css_width(self, css_width: str): """Set the width of the canvas as a CSS string.""" - asgi.send({"type": "comm-css-width", "value": css_width}) + asgi.send_all({"type": "comm-css-width", "value": css_width}) def set_css_height(self, css_height: str): """Set the height of the canvas as a CSS string.""" - asgi.send({"type": "comm-css-height", "value": css_height}) + asgi.send_all({"type": "comm-css-height", "value": css_height}) asgi = Asgi(resources) From aef94fd31e770ad470e94db238e32ae20ee6dccb Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Thu, 9 Apr 2026 15:37:44 +0200 Subject: [PATCH 04/12] clean js and use binary frames instead of base64 --- examples/cube_auto.py | 2 +- rendercanvas/core/renderview-client.js | 264 +++++++++++++------------ rendercanvas/http.py | 2 +- 3 files changed, 139 insertions(+), 129 deletions(-) diff --git a/examples/cube_auto.py b/examples/cube_auto.py index 15ebe5c6..adfd4a7c 100644 --- a/examples/cube_auto.py +++ b/examples/cube_auto.py @@ -7,7 +7,7 @@ # run_example = true -from rendercanvas.auto import RenderCanvas, loop +from rendercanvas.http import RenderCanvas, loop from rendercanvas.utils.cube import setup_drawing_sync canvas = RenderCanvas( diff --git a/rendercanvas/core/renderview-client.js b/rendercanvas/core/renderview-client.js index 43f3f2d0..604beefc 100644 --- a/rendercanvas/core/renderview-client.js +++ b/rendercanvas/core/renderview-client.js @@ -1,75 +1,15 @@ -/* Code to use renderview in a remote browser (http backend). Some of the logic is very similar to 'renderview-afm.js'. */ +/************************************************************************************************* + renderview-client.js -/* global BaseRenderView WebSocket */ + Code to use renderview in a remote browser (rendercanvas http backend). + There are basically two approaches to take. Either use renderview-afm.js and re-use the render logic, + but implement an AFM host. Or directly attach a RenderView to a websocket. I went for the latter. Even + though that means duplicating some code, it looks like this leads to simpler and shorter code. -class ClientRenderView extends BaseRenderView { - constructor (wrapperElement) { - wrapperElement.classList.add('renderview-wrapper') - - // Create view element - const viewElement = document.createElement('img') - viewElement.decoding = 'sync' - viewElement.loading = 'eager' - viewElement.style.touchAction = 'none' // prevent default pan/zoom behavior - viewElement.ondragstart = () => false // prevent browser's built-in image drag - - // Instantiate - super(viewElement, wrapperElement) - this.setThrottle(20) // 20ms -> max 50 move/wheel events per second - - this.frames = [] - this.imgUpdatePending = false - this.lastSrc = null - } - - onEvent (event) { - if (websocket !== null) { - websocket.send(JSON.stringify(event)) - } - } - - requestAnimationFrame () { - // Request an animation frame. - // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, - // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. - if (!this.imgUpdatePending) { - this.imgUpdatePending = true - window.requestAnimationFrame(this.animate.bind(this)) - } - } - - animate () { - this.imgUpdatePending = false - if (this.frames.length === 0) { return }; - - // Pick the oldest frame from the stack, and get its source - const frame = this.frames.shift() - let newSrc - if (frame.buffers && frame.buffers.length > 0) { - const blob = new Blob([frame.buffers[0].buffer], { type: frame.mimetype }) - newSrc = URL.createObjectURL(blob) - } else { - newSrc = frame.data_b64 - } - - // Revoke last objectURL - URL.revokeObjectURL(this.lastSrc) - this.lastSrc = newSrc + *************************************************************************************************/ - // Update the image sources - view.viewElement.src = newSrc - view.viewElement.onload = this.requestAnimationFrame.bind(this) - - // Let the server know we processed the image (even if it's not shown yet) - this.sendResponse(frame) - } +/* global BaseRenderView WebSocket */ - sendResponse (frame) { - // Let Python know what we have at the frame. - const event = { type: '_framefeedback', index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } - this.onEvent(event) - } -} const wrapperElement = document.getElementById('canvas') const statusElement = document.getElementById('status') @@ -80,74 +20,144 @@ updateStatus() openWebsocketConnection() window.openWebsocketConnection = openWebsocketConnection -function updateStatus () { - if (statusElement === null) { return } - if (websocket === null) { - statusElement.innerHTML = "? Disconnected " - } else { - statusElement.innerHTML = "+ Connected" - } +class ClientRenderView extends BaseRenderView { + constructor(wrapperElement) { + wrapperElement.classList.add('renderview-wrapper') + + // Create view element + const viewElement = document.createElement('img') + viewElement.decoding = 'sync' + viewElement.loading = 'eager' + viewElement.style.touchAction = 'none' // prevent default pan/zoom behavior + viewElement.ondragstart = () => false // prevent browser's built-in image drag + + // Instantiate + super(viewElement, wrapperElement) + this.setThrottle(20) // 20ms -> max 50 move/wheel events per second + + this.frames = [] + this.imgUpdatePending = false + this.lastSrc = null + } + + onEvent(event) { + if (websocket !== null) { + websocket.send(JSON.stringify(event)) + } + } + + requestAnimationFrame() { + // Request an animation frame. + // Before the anywidget refactor, we did this via a tiny delay, which supposedly made things more smooth, + // but it also increases the delay for a frame to hit the screen, and limits the max fps, so let's not do that. + if (!this.imgUpdatePending) { + this.imgUpdatePending = true + window.requestAnimationFrame(this.animate.bind(this)) + } + } + + animate() { + this.imgUpdatePending = false + if (this.frames.length === 0) { return }; + + // Pick the oldest frame from the stack, and get its source + const frame = this.frames.shift() + let newSrc + if (frame.buffers && frame.buffers.length > 0) { + const blob = new Blob([frame.buffers[0]], { type: frame.mimetype }) + newSrc = URL.createObjectURL(blob) + } else { + newSrc = frame.data_b64 + } + + // Revoke last objectURL + URL.revokeObjectURL(this.lastSrc) + this.lastSrc = newSrc + + // Update the image sources + view.viewElement.src = newSrc + view.viewElement.onload = this.requestAnimationFrame.bind(this) + + // Let the server know we processed the image (even if it's not shown yet) + this.sendResponse(frame) + } + + sendResponse(frame) { + // Let Python know what we have at the frame. + const event = { type: '_framefeedback', index: frame.index, timestamp: frame.timestamp, localtime: Date.now() / 1000 } + this.onEvent(event) + } } -function openWebsocketConnection () { - const ws = new WebSocket('ws://' + window.location.host + '/ws') +function updateStatus() { + if (statusElement === null) { return } - ws.onopen = (e) => { - console.log('websocket opened') - websocket = ws - window.websocket = ws // allow manual closing to mimic lost connection - if (view === null) { - view = new ClientRenderView(wrapperElement) - console.log('created ClientRenderView') + if (websocket === null) { + statusElement.innerHTML = "? Disconnected " + } else { + statusElement.innerHTML = "+ Connected" } - updateStatus() - } - ws.onerror = (e) => { - console.log(`websocket error: ${e}`) - websocket = null - updateStatus() - } - - let pendingMsg - ws.onmessage = (e) => { - let msg = null - - // First some handling to support a message with buffers - if (typeof e.data === 'string' || e.data instanceof String) { - msg = JSON.parse(e.data) - if (msg.nbuffers && msg.nbuffers > 0) { - pendingMsg = msg - pendingMsg.buffers = [] - msg = null - } else { - pendingMsg = null // discard unfinished pending message (if any) - } - } else { // Blob - if (pendingMsg !== null) { - pendingMsg.buffers.push(e.data) - if (pendingMsg.buffers.length >= pendingMsg.nbuffers) { - msg = pendingMsg - pendingMsg = null +} + +function openWebsocketConnection() { + const ws = new WebSocket('ws://' + window.location.host + '/ws') + + ws.onopen = (e) => { + console.log('websocket opened') + websocket = ws + window.websocket = ws // allow manual closing to mimic lost connection + if (view === null) { + view = new ClientRenderView(wrapperElement) + console.log('created ClientRenderView') } - } + updateStatus() + } + ws.onerror = (e) => { + console.log(`websocket error: ${e}`) + websocket = null + updateStatus() } - if (msg === null) { return } + let pendingMsg + ws.onmessage = (e) => { + let msg = null + + // First some handling to support a message with buffers + if (typeof e.data === 'string' || e.data instanceof String) { + msg = JSON.parse(e.data) + if (msg.nbuffers && msg.nbuffers > 0) { + pendingMsg = msg + pendingMsg.buffers = [] + msg = null + } else { + pendingMsg = null // discard unfinished pending message (if any) + } + } else { // Blob + if (pendingMsg !== null) { + pendingMsg.buffers.push(e.data) + if (pendingMsg.buffers.length >= pendingMsg.nbuffers) { + msg = pendingMsg + pendingMsg = null + } + } + } + + if (msg === null) { return } - // Process message - // console.log(msg) - if (msg.type === 'framebufferdata') { - view.frames.push(msg) - view.requestAnimationFrame() - } else if (msg.type === 'cursor') { - view.setCursor(msg.value) + // Process message + // console.log(msg) + if (msg.type === 'framebufferdata') { + view.frames.push(msg) + view.requestAnimationFrame() + } else if (msg.type === 'cursor') { + view.setCursor(msg.value) + } } - } - ws.onclose = (e) => { - console.log(`websocket closed: ${e.reason} (${e.code})`) - websocket = null - updateStatus() - } + ws.onclose = (e) => { + console.log(`websocket closed: ${e.reason} (${e.code})`) + websocket = null + updateStatus() + } } diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 0a677219..864b66bf 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -370,7 +370,7 @@ def _send_frame(self, array, is_lossless_redraw=False): # Turn array into a based64-encoded JPEG or PNG t1 = time.perf_counter() mimetype, data = encode_array(array, quality) - if False: # use_websocket: + if True: # use_websocket: buffers = [data] data_b64 = None else: From e6aea2fe007439c711e40cd981f18811cf292d98 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Thu, 9 Apr 2026 16:23:20 +0200 Subject: [PATCH 05/12] fix for uviloop --- rendercanvas/utils/asyncs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rendercanvas/utils/asyncs.py b/rendercanvas/utils/asyncs.py index 66c954e4..e3ec819b 100644 --- a/rendercanvas/utils/asyncs.py +++ b/rendercanvas/utils/asyncs.py @@ -38,6 +38,8 @@ def detect_current_async_lib(): libname = "rendercanvas.utils.asyncadapter" elif libname == "pyodide": libname = "asyncio" + elif libname == "uvloop": + libname = "asyncio" return libname From 0bb5929d9cfdf7b1180392c6dc75fde1a429402c Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Thu, 9 Apr 2026 16:26:44 +0200 Subject: [PATCH 06/12] Allow mounting in a larger app --- rendercanvas/core/renderview-client.js | 2 +- rendercanvas/http.py | 33 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/rendercanvas/core/renderview-client.js b/rendercanvas/core/renderview-client.js index 604beefc..1f2198a7 100644 --- a/rendercanvas/core/renderview-client.js +++ b/rendercanvas/core/renderview-client.js @@ -101,7 +101,7 @@ function updateStatus() { } function openWebsocketConnection() { - const ws = new WebSocket('ws://' + window.location.host + '/ws') + const ws = new WebSocket('ws://' + window.location.host + window.location.pathname) ws.onopen = (e) => { console.log('websocket opened') diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 864b66bf..46dd4afd 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -50,11 +50,11 @@ def _load_resource(fname): # A dict with resources to serve. It maps path -> (content-type, body) resources = {} -resources["/"] = "text/html", HTML -resources["/index.html"] = "text/html", HTML -resources["/renderview.css"] = "text/css", _load_resource("renderview.css") +resources[""] = "text/html", HTML +resources["index.html"] = "text/html", HTML +resources["renderview.css"] = "text/css", _load_resource("renderview.css") for fname in ("renderview.js", "renderview-afm.js", "renderview-client.js"): - resources[f"/{fname}"] = "text/javascript", _load_resource(fname) + resources[fname] = "text/javascript", _load_resource(fname) class Websocket: @@ -114,7 +114,7 @@ def send(self, data): data = data else: RuntimeError("ws.send expects dict or bytes") - asyncio.create_task(self._send_queue.put(data)) + asyncio.create_task(self._send_queue.put(data)) # noqa: RUF006 def close(self): """Close the websocket from our end.""" @@ -148,12 +148,7 @@ async def __call__(self, scope, receive, send): while True: message = await receive() if message["type"] == "lifespan.startup": - try: - asyncio.get_running_loop().create_task(loop._rc_run_async()) - except Exception as err: - print("could not start rendercanvas loop:", err) - else: - print("rendercanvas loop started in server") + loop.kickstart() await send({"type": "lifespan.startup.complete"}) elif message["type"] == "lifespan.shutdown": ... # Do some shutdown here! @@ -161,7 +156,9 @@ async def __call__(self, scope, receive, send): return elif scope["type"] == "http": - content_type_and_body = self._resources.get(scope["path"], None) + # Just assume a flat resources dict, so we can mount anywhere in a larger app + fname = scope["path"].rsplit("/", 1)[-1] + content_type_and_body = self._resources.get(fname, None) if content_type_and_body is not None: content_type, body = content_type_and_body if isinstance(body, str): @@ -187,6 +184,9 @@ async def __call__(self, scope, receive, send): elif scope["type"] == "websocket": await send({"type": "websocket.accept"}) + # When running mounted in a larger app, we miss out on the lifespan events + loop.kickstart() + self._ws_count += 1 ws = Websocket(self, self._ws_count) self._websockets.append(ws) @@ -245,6 +245,15 @@ def _rc_run(self): ] ) + def kickstart(self): + if self._run_loop is None: + try: + asyncio.get_running_loop().create_task(loop._rc_run_async()) + except Exception as err: + print("could not start rendercanvas loop:", err) + else: + print("rendercanvas loop started") + loop = HttpLoop() From c4f16d75d0e26e777d41268a1dfb64dc897ec505 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Thu, 9 Apr 2026 16:27:06 +0200 Subject: [PATCH 07/12] Add examples --- examples/web_cube.py | 22 +++++++++++++++++ examples/web_fastapi.py | 53 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 examples/web_cube.py create mode 100644 examples/web_fastapi.py diff --git a/examples/web_cube.py b/examples/web_cube.py new file mode 100644 index 00000000..41fdbfa7 --- /dev/null +++ b/examples/web_cube.py @@ -0,0 +1,22 @@ +""" +Cube in the browser +------------------- + +Run a wgpu example with the http backend. Also see web_fastapi.py for +how to integrate a rendercanvas into a larger web application. +""" + +# run_example = true + +from rendercanvas.http import RenderCanvas, loop +from rendercanvas.utils.cube import setup_drawing_sync + +canvas = RenderCanvas( + title="The wgpu cube example on $backend", update_mode="continuous" +) +draw_frame = setup_drawing_sync(canvas) +canvas.request_draw(draw_frame) + + +# the loop.run() of this backend uses uvicorn to start a webserver +loop.run() diff --git a/examples/web_fastapi.py b/examples/web_fastapi.py new file mode 100644 index 00000000..7be9beca --- /dev/null +++ b/examples/web_fastapi.py @@ -0,0 +1,53 @@ +""" +FastAPI +------- + +Rendercanvas can do remote rendering as part of a web application. +It implements its own little ASGI application, that can be mounted +as part of a larger web application. This example demonstrates this +with the FastAPI web framework. + +You can now run this like any AGI app, e.g. with uvicorn: + + uvicorn web_fastapi:app + +""" + +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from rendercanvas.http import RenderCanvas, asgi +from rendercanvas.utils.cube import setup_drawing_sync + + +# FastAPI code + +app = FastAPI() + + +@app.get("/", response_class=HTMLResponse) +async def home(): + return """ + + + + Test + + +

Hello world

+

Head over to the rendercanvas client

+ + + """ + + +# Prepare a canvas to render something + +canvas = RenderCanvas( + title="The wgpu cube example on $backend", update_mode="continuous" +) +draw_frame = setup_drawing_sync(canvas) +canvas.request_draw(draw_frame) + + +# Mount rendercanvas in the app +app.mount("/rc", asgi) From 6e74944e5be67aad49de87f83d761838c18d9d89 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 10 Apr 2026 09:44:00 +0200 Subject: [PATCH 08/12] Demonstrate customizing the page --- examples/web_cube.py | 39 +++++++++++++++++++++++++++++++++++++-- rendercanvas/http.py | 8 ++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/examples/web_cube.py b/examples/web_cube.py index 41fdbfa7..4f177f65 100644 --- a/examples/web_cube.py +++ b/examples/web_cube.py @@ -4,12 +4,17 @@ Run a wgpu example with the http backend. Also see web_fastapi.py for how to integrate a rendercanvas into a larger web application. + +This example also shows how the web-page can be customized. """ -# run_example = true +# run_example = false -from rendercanvas.http import RenderCanvas, loop +from rendercanvas.http import RenderCanvas, loop, resources from rendercanvas.utils.cube import setup_drawing_sync +from rendercanvas.core.encoders import encode_png +import numpy as np + canvas = RenderCanvas( title="The wgpu cube example on $backend", update_mode="continuous" @@ -18,5 +23,35 @@ canvas.request_draw(draw_frame) +# Define custom HTML. This is optional. +html = """ + + + RenderCanvas over http + + + + + + + +
+ Loading ... +
+ +
+ + +""" + +# The resources is simply a dict that maps filenames to (content-type, body) tuples. +resources["index.html"] = "text/html", html + + +# You can also add new resources, like images or even extra web pages. +im = np.random.uniform(0, 255, (16, 16, 3)).astype(np.uint8) +resources["logo.png"] = "image/png", encode_png(im) + + # the loop.run() of this backend uses uvicorn to start a webserver loop.run() diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 46dd4afd..89837803 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -33,10 +33,6 @@

Loading ...

-

- - -

 
     
@@ -50,10 +46,9 @@ def _load_resource(fname): # A dict with resources to serve. It maps path -> (content-type, body) resources = {} -resources[""] = "text/html", HTML resources["index.html"] = "text/html", HTML resources["renderview.css"] = "text/css", _load_resource("renderview.css") -for fname in ("renderview.js", "renderview-afm.js", "renderview-client.js"): +for fname in ("renderview.js", "renderview-client.js"): resources[fname] = "text/javascript", _load_resource(fname) @@ -158,6 +153,7 @@ async def __call__(self, scope, receive, send): elif scope["type"] == "http": # Just assume a flat resources dict, so we can mount anywhere in a larger app fname = scope["path"].rsplit("/", 1)[-1] + fname = fname or "index.html" content_type_and_body = self._resources.get(fname, None) if content_type_and_body is not None: content_type, body = content_type_and_body From 06d54fb33bcd3cf0f310feb32fb83ba12f770f91 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 10 Apr 2026 11:29:21 +0200 Subject: [PATCH 09/12] make the code multi-client aware --- examples/web_cube.py | 2 +- rendercanvas/http.py | 132 +++++++++++++++++++++++++++++++------------ 2 files changed, 96 insertions(+), 38 deletions(-) diff --git a/examples/web_cube.py b/examples/web_cube.py index 4f177f65..4a91d10b 100644 --- a/examples/web_cube.py +++ b/examples/web_cube.py @@ -24,7 +24,7 @@ # Define custom HTML. This is optional. -html = """ +html = """ RenderCanvas over http diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 89837803..9835210c 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -73,7 +73,6 @@ async def _websocket_receiver(self, receive): elif "bytes" in event: self._on_receive(event["bytes"]) elif event["type"] == "websocket.disconnect": - # TODO: handle closing from client side break async def _websocket_sender(self, send): @@ -96,12 +95,12 @@ def _on_receive(self, text_or_bytes: str | bytes): except Exception: short_text = text[:100] + "…" if len(text) > 100 else text print(f"Received non-json message: {short_text!r}") + # todo: convert print to log calls return else: - # todo: some messages, like frame feedback, should be processed per-ws, others only by one. - self._app._on_event(self._id, event) + self._app._on_event(event, self._id) - def send(self, data): + def send(self, data: dict | bytes): """Send data into the websocket.""" if isinstance(data, dict): data = json.dumps(data) @@ -116,9 +115,6 @@ def close(self): _ = self._send_queue.put(None) # None means close, see _websocket_sender() -# TODO: how does this work when ppl want to include this in a larger web application, with e.g. FastAPI or Falcon? - - class Asgi: """The ASGI application. @@ -132,9 +128,9 @@ class Asgi: def __init__(self, resources): self._resources = resources - self._websockets = [] - self._event_callback = lambda id, ev: None - self._ws_count = 0 + self._websockets = {} # id -> ws + self._event_callback = lambda ev, id: None + self._ws_counter = 0 async def __call__(self, scope, receive, send): """The ASGI entrypoint.""" @@ -183,10 +179,13 @@ async def __call__(self, scope, receive, send): # When running mounted in a larger app, we miss out on the lifespan events loop.kickstart() - self._ws_count += 1 - ws = Websocket(self, self._ws_count) - self._websockets.append(ws) - self._event_callback(0, {"type": "_new_client"}) + self._ws_counter += 1 + id = self._ws_counter + ws = Websocket(self, id) + self._websockets[id] = ws + self._event_callback( + {"type": "_clients_change", "ids": tuple(self._websockets)}, 0 + ) try: receiver = asyncio.create_task(ws._websocket_receiver(receive)) @@ -198,24 +197,38 @@ async def __call__(self, scope, receive, send): for task in pending: task.cancel() finally: - self._websockets.remove(ws) + self._websockets.pop(id, None) + self._event_callback( + {"type": "_clients_change", "ids": tuple(self._websockets)}, 0 + ) - def _on_event(self, id, event): + def _on_event(self, event, id): """Called when a websocket receives an event.""" try: - self._event_callback(id, event) + self._event_callback(event, id) except Exception as err: print(f"Error handling ws event callback: {err}") - def send_all(self, data: str | bytes): + def send_all(self, msg: dict): """Send data to all websockets.""" - for ws in self._websockets: - ws.send(data) + assert isinstance(msg, dict) + for ws in self._websockets.values(): + ws.send(msg) + + def send_to(self, msg: dict, buffers: list[bytes], ids=list[int]): + if len(buffers) > 0: + assert msg["nbuffers"] == len(buffers) + for id in ids: + ws = self._websockets.get(id, None) + if ws is not None: + ws.send(msg) + for buffer in buffers: + ws.send(buffer) def close(self): """Disconnect all clients.""" # TODO: also put in a closed (non-restartable) state? i.e. think about lifetime cycle - for ws in self._websockets: + for ws in self._websockets.values(): ws.close() def get_count(self): @@ -264,6 +277,7 @@ class HttpRenderCanvas(BaseRenderCanvas): _rc_canvas_group = HttpCanvasGroup(loop) _max_buffered_frames = 2 + _quality = 80 def __init__(self, *args, **kwargs): @@ -281,33 +295,47 @@ def __init__(self, *args, **kwargs): self._warned_png = False self._lossless_draw_info = None + self._active_client = 0 + self._confirmed_frame_per_client = {} + self.reset_stats() # Set size, title, etc. self._final_canvas_init() - def _on_event(self, id: int, event: dict): + def _on_event(self, event: dict, id: int): try: type = event["type"] except KeyError: logger.warning(f"Invalid event: {event!r}") return - # TODO: some logic depends on the id - # TODO: keep track of frame feedback per id, main ws determines frame rate, others drop frames as necessary - if type.startswith("_"): - if type == "_framefeedback": - self._frame_feedback = event - loop.call_soon(self._maybe_draw) - elif type == "_new_client": + # Internal event + if type == "_clients_change": + # Update our per-client info + self._confirmed_frame_per_client = { + id: self._confirmed_frame_per_client.get(id, 0) + for id in event["ids"] + } + print(self._confirmed_frame_per_client) + # select longest connected client as the new active one + self._active_client = event["ids"][0] # Force a draw. With a new client, we want to override frame feedback self._draw_requested = 2 loop.call_soon(self._maybe_draw) + elif type == "_framefeedback": + # Update last confirmed frame. But only schedule new draws based on the active client. + self._confirmed_frame_per_client[id] = event["index"] + if id == self._active_client: + self._frame_feedback = event + loop.call_soon(self._maybe_draw) else: - logger.warning(f"Unknown comm event: {event!r}") + logger.warning(f"Unknown event: {event!r}") else: # A renderview event + if id != self._active_client: + return # ignore events from passive clients if type == "resize": self._size_info.set_physical_size( @@ -328,14 +356,22 @@ def _on_event(self, id: int, event: dict): def _maybe_draw(self): """Perform a draw, if we can and should.""" + # TODO: restore _schedule_maybe_draw? bc there's quite a lot of invocations with multiple clients feedback = self._frame_feedback # Update stats self._update_stats(feedback) + # Determine frames in flight for all clients, allowing 2 more in-flight + need_index = self._frame_index - self._max_buffered_frames + 1 - 2 + all_clients_ok = all( + index == 0 or index >= need_index + for index in self._confirmed_frame_per_client.values() + ) # Determine whether we should perform a draw: a draw was requested, and # the client is ready for a new frame, and the client widget is visible. frames_in_flight = self._frame_index - feedback.get("index", 0) should_draw = ( self._draw_requested + and all_clients_ok and ( frames_in_flight < self._max_buffered_frames or self._draw_requested >= 2 @@ -416,9 +452,30 @@ def _send_frame(self, array, is_lossless_redraw=False): index=self._frame_index, timestamp=timestamp, ) - asgi.send_all(msg) - for buffer in buffers: - asgi.send_all(buffer) + + # Which client receive this? + ids = list(self._confirmed_frame_per_client.keys()) + + # Currently we send each frame to all clients, and also throttle on the slowest + # client (though allowing more in-flight frames for passive clients). This means + # that one slow client means a low framerate for all clients. Alternatively, we + # can throttle on the active client, and drop frames for the passive clients + # that are not up-to-date enough. The commented code below does that. It is not + # fully functional, bc the frames_in_flight is over-estimated bc we drop frames + # ;) In any case, when/if we use diff images each client MUST have each frame. + # Therefore I kept the simpler throttle-on-all approach. + # + # ids = [self._active_client] + # for id, confirmed_index in self._confirmed_frame_per_client.items(): + # if id != self._active_client: + # # Add id if it has not too many frames in flight. Subtract 1 bc we just bumped self._frame_index. Allow 1 extra in-flight frame for passive clients. + # frames_in_flight = self._frame_index - confirmed_index - 1 + # if frames_in_flight < self._max_buffered_frames + 1: + # ids.append(id) + # elif confirmed_index == 0: + # ids.append(id) # new client + + asgi.send_to(msg, buffers, ids) # ----- related to stats @@ -525,18 +582,19 @@ def _rc_get_closed(self): return self._is_closed def _rc_set_title(self, title): - asgi.send_all({"type": "comm-title", "value": title}) + asgi.send_all({"type": "title", "value": title}) def _rc_set_cursor(self, cursor): - asgi.send_all({"type": "comm-cursor", "value": cursor}) + # todo: fix/test this + asgi.send_all({"type": "cursor", "value": cursor}) def set_css_width(self, css_width: str): """Set the width of the canvas as a CSS string.""" - asgi.send_all({"type": "comm-css-width", "value": css_width}) + asgi.send_all({"type": "css_width", "value": css_width}) def set_css_height(self, css_height: str): """Set the height of the canvas as a CSS string.""" - asgi.send_all({"type": "comm-css-height", "value": css_height}) + asgi.send_all({"type": "css_height", "value": css_height}) asgi = Asgi(resources) From b42f4ad42e00a6b0622680291452597c772fe01a Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 10 Apr 2026 12:23:25 +0200 Subject: [PATCH 10/12] shown in status whether active or passive --- rendercanvas/core/renderview-client.js | 11 ++++++++++- rendercanvas/http.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rendercanvas/core/renderview-client.js b/rendercanvas/core/renderview-client.js index 1f2198a7..c289b7e3 100644 --- a/rendercanvas/core/renderview-client.js +++ b/rendercanvas/core/renderview-client.js @@ -15,6 +15,7 @@ const wrapperElement = document.getElementById('canvas') const statusElement = document.getElementById('status') let view = null let websocket = null +let isActive = null updateStatus() openWebsocketConnection() @@ -93,10 +94,15 @@ class ClientRenderView extends BaseRenderView { function updateStatus() { if (statusElement === null) { return } + let activeText = '' + if (isActive !== null) { + activeText = isActive ? ' (active)' : '(passive)' + } + if (websocket === null) { statusElement.innerHTML = "? Disconnected " } else { - statusElement.innerHTML = "+ Connected" + statusElement.innerHTML = `+ Connected ${activeText}` } } @@ -150,6 +156,9 @@ function openWebsocketConnection() { if (msg.type === 'framebufferdata') { view.frames.push(msg) view.requestAnimationFrame() + } else if (msg.type === 'active') { + isActive = msg.value + updateStatus() } else if (msg.type === 'cursor') { view.setCursor(msg.value) } diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 9835210c..340b36db 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -321,6 +321,7 @@ def _on_event(self, event: dict, id: int): print(self._confirmed_frame_per_client) # select longest connected client as the new active one self._active_client = event["ids"][0] + self._update_active_states() # Force a draw. With a new client, we want to override frame feedback self._draw_requested = 2 loop.call_soon(self._maybe_draw) @@ -596,6 +597,14 @@ def set_css_height(self, css_height: str): """Set the height of the canvas as a CSS string.""" asgi.send_all({"type": "css_height", "value": css_height}) + def _update_active_states(self): + active_ids = [self._active_client] + passive_ids = set(self._confirmed_frame_per_client.keys()) + passive_ids.discard(self._active_client) + + asgi.send_to({"type": "active", "value": True}, [], active_ids) + asgi.send_to({"type": "active", "value": False}, [], passive_ids) + asgi = Asgi(resources) RenderCanvas = HttpRenderCanvas From e45e2a21dab046a578025750baba65ce894c775d Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 29 May 2026 13:09:10 +0200 Subject: [PATCH 11/12] Small tweaks, renames, and fixes for closing connections --- examples/cube_auto.py | 2 +- examples/{web_cube.py => cube_http.py} | 8 ++-- examples/{web_fastapi.py => fastapi_app.py} | 2 +- rendercanvas/core/renderview-client.js | 2 +- rendercanvas/http.py | 50 +++++++++++++-------- 5 files changed, 39 insertions(+), 25 deletions(-) rename examples/{web_cube.py => cube_http.py} (82%) rename examples/{web_fastapi.py => fastapi_app.py} (97%) diff --git a/examples/cube_auto.py b/examples/cube_auto.py index adfd4a7c..15ebe5c6 100644 --- a/examples/cube_auto.py +++ b/examples/cube_auto.py @@ -7,7 +7,7 @@ # run_example = true -from rendercanvas.http import RenderCanvas, loop +from rendercanvas.auto import RenderCanvas, loop from rendercanvas.utils.cube import setup_drawing_sync canvas = RenderCanvas( diff --git a/examples/web_cube.py b/examples/cube_http.py similarity index 82% rename from examples/web_cube.py rename to examples/cube_http.py index 4a91d10b..822a02e3 100644 --- a/examples/web_cube.py +++ b/examples/cube_http.py @@ -2,10 +2,12 @@ Cube in the browser ------------------- -Run a wgpu example with the http backend. Also see web_fastapi.py for -how to integrate a rendercanvas into a larger web application. +Run a wgpu example with the http backend. Note that the http backend can be used +with most examples by simply using ``from rendercanvas.http import RenderCanvas, +loop``. This example also shows how the web-page can be customized. -This example also shows how the web-page can be customized. +Also see fastapi_app.py for how to integrate a rendercanvas into a larger web +application. """ # run_example = false diff --git a/examples/web_fastapi.py b/examples/fastapi_app.py similarity index 97% rename from examples/web_fastapi.py rename to examples/fastapi_app.py index 7be9beca..4bb85501 100644 --- a/examples/web_fastapi.py +++ b/examples/fastapi_app.py @@ -9,7 +9,7 @@ You can now run this like any AGI app, e.g. with uvicorn: - uvicorn web_fastapi:app + uvicorn fastapi_app:app """ diff --git a/rendercanvas/core/renderview-client.js b/rendercanvas/core/renderview-client.js index c289b7e3..3d00e17a 100644 --- a/rendercanvas/core/renderview-client.js +++ b/rendercanvas/core/renderview-client.js @@ -4,7 +4,7 @@ Code to use renderview in a remote browser (rendercanvas http backend). There are basically two approaches to take. Either use renderview-afm.js and re-use the render logic, but implement an AFM host. Or directly attach a RenderView to a websocket. I went for the latter. Even - though that means duplicating some code, it looks like this leads to simpler and shorter code. + though that means duplicating some code, it looks like this leads to simpler code. *************************************************************************************************/ diff --git a/rendercanvas/http.py b/rendercanvas/http.py index 340b36db..f31cde98 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -1,5 +1,5 @@ """ -A remote backend one or more browser views. +A remote backend with one or more browser views. This module implements an ASGI web application, so it runs on any ASGI server. We default to uvicorn. """ @@ -65,25 +65,35 @@ def __init__(self, app, id): self._send_queue = asyncio.Queue() async def _websocket_receiver(self, receive): - while True: - event = await receive() # asgi event - if event["type"] == "websocket.receive": - if "text" in event: - self._on_receive(event["text"]) - elif "bytes" in event: - self._on_receive(event["bytes"]) - elif event["type"] == "websocket.disconnect": - break + try: + while True: + event = await receive() # asgi event + if event["type"] == "websocket.receive": + if "text" in event: + self._on_receive(event["text"]) + elif "bytes" in event: + self._on_receive(event["bytes"]) + elif event["type"] == "websocket.disconnect": + break + except asyncio.CancelledError: + pass async def _websocket_sender(self, send): - while True: - msg = await self._send_queue.get() - if msg is None: - await send({"type": "websocket.close", "code": 1000}) - elif isinstance(msg, str): - await send({"type": "websocket.send", "text": msg}) - else: - await send({"type": "websocket.send", "bytes": msg}) + try: + while True: + msg = await self._send_queue.get() + if msg is None: + await send({"type": "websocket.close", "code": 1000}) + break + elif isinstance(msg, str): + await send({"type": "websocket.send", "text": msg}) + else: + await send({"type": "websocket.send", "bytes": msg}) + except asyncio.CancelledError: + pass + except Exception as err: + if "disconnect" not in err.__class__.__name__.lower(): + raise err from None def _on_receive(self, text_or_bytes: str | bytes): if isinstance(text_or_bytes, bytes): @@ -242,6 +252,8 @@ def run(self, host="localhost", port=60649): return super().run() def _rc_run(self): + # Allow the standard rendercanvas usage (``loop.run()``) to start the web server + from uvicorn.main import main as uvicorn_main print(f"Starting server at http://{self._host}:{self._port}") @@ -320,7 +332,7 @@ def _on_event(self, event: dict, id: int): } print(self._confirmed_frame_per_client) # select longest connected client as the new active one - self._active_client = event["ids"][0] + self._active_client = event["ids"][0] if event["ids"] else 0 self._update_active_states() # Force a draw. With a new client, we want to override frame feedback self._draw_requested = 2 From 451716e63e13fc024210f61a92860f02dcc7e441 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 29 May 2026 13:23:56 +0200 Subject: [PATCH 12/12] Some cleanup --- rendercanvas/http.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/rendercanvas/http.py b/rendercanvas/http.py index f31cde98..33e2f425 100644 --- a/rendercanvas/http.py +++ b/rendercanvas/http.py @@ -97,15 +97,14 @@ async def _websocket_sender(self, send): def _on_receive(self, text_or_bytes: str | bytes): if isinstance(text_or_bytes, bytes): - print("Unexpectedly received bytes ({len(msg}).") + logger.warning("Unexpectedly received bytes ({len(msg}).") else: text = text_or_bytes try: event = json.loads(text) # JS event except Exception: short_text = text[:100] + "…" if len(text) > 100 else text - print(f"Received non-json message: {short_text!r}") - # todo: convert print to log calls + logger.warning(f"Received non-json message: {short_text!r}") return else: self._app._on_event(event, self._id) @@ -217,7 +216,7 @@ def _on_event(self, event, id): try: self._event_callback(event, id) except Exception as err: - print(f"Error handling ws event callback: {err}") + logger.warning(f"Error handling ws event callback: {err}") def send_all(self, msg: dict): """Send data to all websockets.""" @@ -237,7 +236,7 @@ def send_to(self, msg: dict, buffers: list[bytes], ids=list[int]): def close(self): """Disconnect all clients.""" - # TODO: also put in a closed (non-restartable) state? i.e. think about lifetime cycle + # I guess technically clients can reconnect again. Not sure if that works. for ws in self._websockets.values(): ws.close() @@ -256,7 +255,8 @@ def _rc_run(self): from uvicorn.main import main as uvicorn_main - print(f"Starting server at http://{self._host}:{self._port}") + # Use warning level; if using info, the message may not be shown + logger.warning(f"Starting server at http://{self._host}:{self._port}") uvicorn_main( [ f"--host={self._host}", @@ -271,9 +271,9 @@ def kickstart(self): try: asyncio.get_running_loop().create_task(loop._rc_run_async()) except Exception as err: - print("could not start rendercanvas loop:", err) + logger.error("could not start rendercanvas loop:", err) else: - print("rendercanvas loop started") + logger.info("rendercanvas loop started") loop = HttpLoop() @@ -295,7 +295,7 @@ class HttpRenderCanvas(BaseRenderCanvas): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # todo: limit to a single canvas + # Note: we assume there is only a sinle canvas asgi._event_callback = self._on_event self._is_closed = False @@ -598,7 +598,6 @@ def _rc_set_title(self, title): asgi.send_all({"type": "title", "value": title}) def _rc_set_cursor(self, cursor): - # todo: fix/test this asgi.send_all({"type": "cursor", "value": cursor}) def set_css_width(self, css_width: str):