From cfa385995b58d8307b17f8c2614cbd63107cdab9 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:36:47 +0200
Subject: [PATCH 01/19] Namespaced topics, stream teardown, embeddable broker

M0.2/M0.4 plus the transport half of M0.1/M0.5.

- PubSubClient takes a mandatory namespace; topics become
  `dt/<namespace>/dtypes/<label>` so twins which use identical dtype
  labels no longer cross-subscribe on a shared broker.  A topic
  terminator keeps a label from prefix-matching a longer one (ZMQ
  SUBSCRIBE is a prefix match; hygiene, not correctness).
- Teardown: ZMQ_PS_Client.close() cancels the receive loop, closes the
  sockets and terminates the context; PubSubClient.unsubscribe_dtype()
  and close() drop subscriptions and the backend with them.  The
  connect monitor is now attached before connecting (it only reports
  later events) and is explicitly closed -- disable_monitor() only
  detaches it, and the leftover socket blocked ctx.term() forever.
- ZMQ_Broker binds in the process that runs the proxy (a context does
  not survive spawn), defaults to a random loopback port and reports
  what it bound.  ZMQ_BrokerProcess embeds it as a spawn-context
  subprocess with async start/stop; the blocking spawn/join calls run
  off the event loop.
- New `config` module: the one place that decides addresses.
  Loopback-only defaults (R7), env overridable.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/config.py    |  43 ++++
 src/digitaltwin/streaming.py | 386 +++++++++++++++++++++++++----------
 2 files changed, 326 insertions(+), 103 deletions(-)
 create mode 100644 src/digitaltwin/config.py

diff --git a/src/digitaltwin/config.py b/src/digitaltwin/config.py
new file mode 100644
index 0000000..c72bfe1
--- /dev/null
+++ b/src/digitaltwin/config.py
@@ -0,0 +1,43 @@
+"""Deployment configuration for the digital twin framework.
+
+The single place where stream endpoint addresses are decided.  No other
+module (and no demo) may contain a hardcoded transport address.
+
+Binding policy: the framework binds to loopback unless the deployment
+explicitly configures something else.  The pubsub data plane carries
+cloudpickled payloads, so anyone who can reach the broker ports can
+execute code in every subscriber -- non-loopback binds require an
+explicit configuration and a firewalled/private network.
+"""
+
+import os
+
+# loopback-only by default -- see the binding policy above
+DEFAULT_BIND_HOST = "127.0.0.1"
+
+# fixed default ports, used by the standalone broker (two-terminal demos)
+DEFAULT_PUB_ADDR = f"tcp://{DEFAULT_BIND_HOST}:5000"
+DEFAULT_SUB_ADDR = f"tcp://{DEFAULT_BIND_HOST}:5001"
+
+# wildcard port: let the OS pick.  Used by embedded (subprocess) brokers,
+# which report their bound addresses back to the parent.
+RANDOM_PUB_ADDR = f"tcp://{DEFAULT_BIND_HOST}:*"
+RANDOM_SUB_ADDR = f"tcp://{DEFAULT_BIND_HOST}:*"
+
+ENV_PUB_ADDR = "DT_STREAM_PUB_ADDR"
+ENV_SUB_ADDR = "DT_STREAM_SUB_ADDR"
+
+
+def stream_addresses(
+    pub_addr: str | None = None, sub_addr: str | None = None
+) -> tuple[str, str]:
+    """Resolve the (publish, subscribe) addresses of the stream broker.
+
+    Precedence: explicit argument, then the `DT_STREAM_PUB_ADDR` /
+    `DT_STREAM_SUB_ADDR` environment variables, then the loopback defaults.
+    """
+
+    return (
+        pub_addr or os.environ.get(ENV_PUB_ADDR) or DEFAULT_PUB_ADDR,
+        sub_addr or os.environ.get(ENV_SUB_ADDR) or DEFAULT_SUB_ADDR,
+    )
diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index 4bbc211..b2c2746 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -2,19 +2,29 @@
 
 ## Actual streaming backend is pluggable.
 
-from abc import ABC, abstractmethod
 import asyncio
+import contextlib
+import logging
+import multiprocessing
+
+from abc import ABC, abstractmethod
 from typing import Callable, Optional
 
 import cloudpickle
-from .components import DataType, TypedData
 import zmq
 import zmq.asyncio
+
 from zmq.utils.monitor import recv_monitor_message
-import logging
+
+from .components import DataType, TypedData
+from .config import RANDOM_PUB_ADDR, RANDOM_SUB_ADDR, stream_addresses
 
 logger = logging.getLogger(__name__)
 
+# bounded waits: no teardown path may hang the host event loop
+BROKER_START_TIMEOUT = 30.0
+BROKER_STOP_TIMEOUT = 5.0
+
 
 class PubSubBackend(ABC):
     label = "generic"
@@ -38,39 +48,161 @@ async def subscribe(self, topic, callback, **kwargs):
     async def unsubscribe(self, topic):
         pass
 
+    @abstractmethod
+    async def close(self):
+        """Release all resources.  Idempotent."""
+
     def __str__(self):
         return f"{self.label}"
 
 
-class MQTTBackend:
-    label = "mqtt"
-    pass
-
-
 # Use ZMQ for the broker
 class ZMQ_Broker:
-    def __init__(self, publish_addr: str, subscribe_addr: str):
-        self.ctx = zmq.Context()
-        self.pub_recv = zmq.Socket(self.ctx, zmq.XSUB)
-        self.sub_send = zmq.Socket(self.ctx, zmq.XPUB)
+    """XSUB/XPUB proxy.  `run()` blocks -- it is meant to own its process.
 
-        self.publish_addr = publish_addr
-        self.subscribe_addr = subscribe_addr
+    Addresses default to a random port on loopback (see `config`); the
+    actually bound addresses are available from `get_connection_str()`
+    once `bind()` ran.
+    """
 
-    def run(self):
+    def __init__(
+        self, publish_addr: Optional[str] = None, subscribe_addr: Optional[str] = None
+    ):
+        self.publish_addr = publish_addr or RANDOM_PUB_ADDR
+        self.subscribe_addr = subscribe_addr or RANDOM_SUB_ADDR
+
+        self.ctx: Optional[zmq.Context] = None
+        self.pub_recv: Optional[zmq.Socket] = None
+        self.sub_send: Optional[zmq.Socket] = None
+
+    def bind(self) -> tuple[str, str]:
+        """Create the sockets and bind them.  Returns the bound addresses.
+
+        Must run in the process that will run the proxy -- a ZMQ context
+        does not survive a fork/spawn.
+        """
+
+        self.ctx = zmq.Context()
+        self.pub_recv = self.ctx.socket(zmq.XSUB)
+        self.sub_send = self.ctx.socket(zmq.XPUB)
 
         self.pub_recv.bind(self.publish_addr)
         self.sub_send.bind(self.subscribe_addr)
 
-        zmq.proxy(self.pub_recv, self.sub_send)
+        # resolve wildcard ports to what the OS actually handed out
+        self.publish_addr = self.pub_recv.getsockopt_string(zmq.LAST_ENDPOINT)
+        self.subscribe_addr = self.sub_send.getsockopt_string(zmq.LAST_ENDPOINT)
+
+        return self.get_connection_str()
 
-        self.pub_recv.close()
-        self.sub_send.close()
-        self.ctx.term()
+    def run(self):
+        if self.ctx is None:
+            self.bind()
+
+        try:
+            zmq.proxy(self.pub_recv, self.sub_send)
+        except zmq.ContextTerminated:
+            pass
+        finally:
+            self.pub_recv.close(linger=0)
+            self.sub_send.close(linger=0)
+            self.ctx.term()
+
+    def get_connection_str(self) -> tuple[str, str]:
+        return self.publish_addr, self.subscribe_addr
+
+
+def _broker_main(publish_addr, subscribe_addr, conn):
+    """Entry point of the broker subprocess (must be importable for spawn)."""
+
+    broker = ZMQ_Broker(publish_addr, subscribe_addr)
+    conn.send(broker.bind())
+    conn.close()
+    broker.run()
+
+
+class ZMQ_BrokerProcess:
+    """A `ZMQ_Broker` embedded as a spawn-context subprocess.
+
+    The subprocess boundary is the stop path: `zmq.proxy()` has none of
+    its own.  The child binds (random port by default), reports the bound
+    addresses back to the parent, and is stopped by terminate/join.
+    """
+
+    def __init__(
+        self, publish_addr: Optional[str] = None, subscribe_addr: Optional[str] = None
+    ):
+        self._addrs = (publish_addr or RANDOM_PUB_ADDR, subscribe_addr or RANDOM_SUB_ADDR)
+        self._proc: Optional[multiprocessing.process.BaseProcess] = None
+
+        self.publish_addr: Optional[str] = None
+        self.subscribe_addr: Optional[str] = None
+
+    async def start(self, timeout: float = BROKER_START_TIMEOUT) -> tuple[str, str]:
+        """Spawn the broker and return its bound (publish, subscribe) pair."""
 
-    def get_connection_str(self):
+        if self._proc is not None:
+            return self.get_connection_str()
+
+        # spawn + pipe read are blocking: keep them off the event loop
+        addrs = await asyncio.to_thread(self._start, timeout)
+        self.publish_addr, self.subscribe_addr = addrs
+        logger.info("stream broker at %s / %s", *addrs)
+
+        return addrs
+
+    async def stop(self, timeout: float = BROKER_STOP_TIMEOUT):
+        """Terminate the broker subprocess.  Idempotent."""
+
+        if self._proc is None:
+            return
+
+        await asyncio.to_thread(self._stop, timeout)
+        self.publish_addr = self.subscribe_addr = None
+
+    def is_alive(self) -> bool:
+        return self._proc is not None and self._proc.is_alive()
+
+    def get_connection_str(self) -> tuple[str, str]:
         return self.publish_addr, self.subscribe_addr
 
+    def _start(self, timeout):
+        ctx = multiprocessing.get_context("spawn")
+        recv_conn, send_conn = ctx.Pipe(duplex=False)
+
+        self._proc = ctx.Process(
+            target=_broker_main, args=(*self._addrs, send_conn), daemon=True
+        )
+        try:
+            self._proc.start()
+            send_conn.close()  # only the child keeps the write end
+
+            if not recv_conn.poll(timeout):
+                raise TimeoutError(f"stream broker did not bind within {timeout}s")
+
+            return recv_conn.recv()
+
+        except BaseException:
+            self._stop(BROKER_STOP_TIMEOUT)
+            raise
+
+        finally:
+            recv_conn.close()
+
+    def _stop(self, timeout):
+        proc, self._proc = self._proc, None
+
+        if proc.is_alive():
+            proc.terminate()
+            proc.join(timeout)
+
+        if proc.is_alive():
+            logger.warning("stream broker ignored terminate -- killing")
+            proc.kill()
+            proc.join(timeout)
+
+        proc.close()
+
 
 class ZMQ_PS_Client(PubSubBackend):
     label = "local"
@@ -81,46 +213,59 @@ def __init__(self, pub_addr: Optional[str] = None, sub_addr: Optional[str] = Non
         self.pub_addr = pub_addr
         self.sub_addr = sub_addr
 
-        ctx = zmq.asyncio.Context()
-        if self.pub_addr is not None:
-            self.pub_soc: Optional[zmq.asyncio.Socket] = ctx.socket(zmq.PUB)
-        else:
-            self.pub_soc = None
-
-        if self.sub_addr is not None:
-            self.sub_soc: Optional[zmq.asyncio.Socket] = ctx.socket(zmq.SUB)
-        else:
-            self.sub_soc = None
+        self._ctx = zmq.asyncio.Context()
+        self.pub_soc: Optional[zmq.asyncio.Socket] = (
+            self._ctx.socket(zmq.PUB) if pub_addr is not None else None
+        )
+        self.sub_soc: Optional[zmq.asyncio.Socket] = (
+            self._ctx.socket(zmq.SUB) if sub_addr is not None else None
+        )
 
         # subscribe: store the callback for the topic
         # publish: send a message to each of the callbacks.
 
-        self.topics: dict[str, Callable] = {}
+        self.topics: dict[str, list[Callable]] = {}
 
-        self.loop: Optional[asyncio.BaseEventLoop] = None
+        self._task: Optional[asyncio.Task] = None
+        self._closed = False
         self.is_running = asyncio.Event()
 
-    async def _wait_for_connect(self, sock):
+    async def _connect_socket(self, sock, addr):
+        """Connect `sock` and wait until the connection is established.
+
+        The monitor is attached *before* connecting: it only reports events
+        which happen after it was attached.
+        """
+
         monitor = sock.get_monitor_socket()
-        while True:
-            event = await recv_monitor_message(monitor)
-            if event["event"] == zmq.EVENT_CONNECTED:
-                break
+        try:
+            sock.connect(addr)
+            while True:
+                event = await recv_monitor_message(monitor)
+                if event["event"] == zmq.EVENT_CONNECTED:
+                    return
+        finally:
+            # also runs on cancellation.  disable_monitor() only detaches
+            # the socket -- leaving it open would block ctx.term()
+            sock.disable_monitor()
+            monitor.close(linger=0)
 
     async def connect(self):
-        if self.sub_addr is not None:
-            self.sub_soc.connect(self.sub_addr)
+        self._check_open()
+
+        if self.sub_soc is not None:
             logger.info("Waiting to connect to ZMQ broker...")
-            await self._wait_for_connect(self.sub_soc)
+            await self._connect_socket(self.sub_soc, self.sub_addr)
+
+        if self.pub_soc is not None:
+            await self._connect_socket(self.pub_soc, self.pub_addr)
 
-        if self.pub_addr is not None:
-            self.pub_soc.connect(self.pub_addr)
-            await self._wait_for_connect(self.pub_soc)
-        self.task = asyncio.create_task(self._run())
+        self._task = asyncio.create_task(self._run())
         await asyncio.sleep(0.1)
         self.is_running.set()
 
     async def publish(self, topic, message):
+        self._check_open()
         if self.pub_soc is None:
             raise ValueError("Publishing endpoint not connected")
 
@@ -133,6 +278,7 @@ async def publish(self, topic, message):
         await self.pub_soc.send_multipart([topic_b, message_b])
 
     async def subscribe(self, topic, callback, **backend_params):
+        self._check_open()
         if self.sub_soc is None:
             raise ValueError("Subscribe endpoint not connected")
 
@@ -141,29 +287,62 @@ async def subscribe(self, topic, callback, **backend_params):
             await self.is_running.wait()
         self.sub_soc.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8"))
 
-        if topic not in self.topics:
-            self.topics[topic] = [callback]
-        else:
-            self.topics[topic].append(callback)
-
-    async def _run(self):
-        if self.sub_soc is None:
-            return
-        while True:
-            topic, message = await self.sub_soc.recv_multipart()
-            item = topic.decode("utf-8")
-            data = cloudpickle.loads(message)
-            for task in self.topics.get(item, []):
-                await task(data)
+        self.topics.setdefault(topic, []).append(callback)
 
     async def unsubscribe(self, topic):
-        if self.sub_soc is None:
-            raise ValueError("Subscribe endpoint not connected")
+        if self._closed or self.sub_soc is None:
+            return
 
         if topic in self.topics:
             self.sub_soc.setsockopt(zmq.UNSUBSCRIBE, topic.encode("utf-8"))
             del self.topics[topic]
 
+    async def close(self):
+        """Cancel the receive loop, close all sockets, terminate the context.
+
+        Idempotent.
+        """
+
+        if self._closed:
+            return
+        self._closed = True
+
+        task, self._task = self._task, None
+        if task is not None:
+            task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await task
+
+        self.topics.clear()
+
+        for sock in (self.pub_soc, self.sub_soc):
+            if sock is not None:
+                sock.close(linger=0)
+        self.pub_soc = self.sub_soc = None
+
+        # all sockets are closed, so this returns immediately
+        self._ctx.term()
+        self.is_running.clear()
+
+    def _check_open(self):
+        if self._closed:
+            raise RuntimeError("stream client is closed")
+
+    async def _run(self):
+        if self.sub_soc is None:
+            return
+        try:
+            while True:
+                topic, message = await self.sub_soc.recv_multipart()
+                item = topic.decode("utf-8")
+                data = cloudpickle.loads(message)
+                for task in self.topics.get(item, []):
+                    await task(data)
+        except asyncio.CancelledError:
+            raise
+        except zmq.ContextTerminated:
+            pass
+
 
 # The pubsub client abstracts away the specifics of the pub / sub
 # implementation. It rather adds the DataType wrapper / connects with the
@@ -171,20 +350,32 @@ async def unsubscribe(self, topic):
 
 
 class PubSubClient:
-    # Topics:
-    # runtime/info/
-    # runtime/dtypes/<dtype_label>
-    RUNTIME_INFO_TOPIC = "runtime/info"
-    RUNTIME_DTYPES = "runtime/dtypes/<dtype_label>"
+    """Namespaced, dtype-aware view on a pubsub backend.
+
+    Topics are `dt/<namespace>/dtypes/<dtype label>`.  The namespace keeps
+    twins that use identical dtype labels apart on a shared broker, so one
+    client per twin is required (also because `subscribe_to_dtype` holds
+    one queue per dtype).
+    """
+
+    # ZMQ SUBSCRIBE is prefix matching: the terminator keeps a label from
+    # matching every label it is a prefix of.  Hygiene, not correctness --
+    # delivery is filtered by exact topic lookup.  A control character is
+    # used because a dtype label may contain any printable character.
+    TOPIC_TERMINATOR = "\x00"
 
     # For now, only support one backend. Future TODO: Add support for multiple backends
 
-    def __init__(self, backend=PubSubBackend):
+    def __init__(self, backend: PubSubBackend, namespace: str):
         self._backend = backend
+        self.namespace = namespace
 
         # so I don't repeat
         self.subscriptions: set[DataType] = set()
 
+    def topic(self, dtype: DataType) -> str:
+        return f"dt/{self.namespace}/dtypes/{dtype.name}{self.TOPIC_TERMINATOR}"
+
     # for runtime use only!!!
     async def subscribe_to_dtype(
         self,
@@ -202,54 +393,43 @@ async def receive_data(message):
             td = TypedData(dtype, message)
             await queue.put(td)
 
-        topic = self.RUNTIME_DTYPES.replace("<dtype_label>", dtype.name)
         await self._backend.subscribe(
-            topic=topic, callback=receive_data, **backend_params
+            topic=self.topic(dtype), callback=receive_data, **backend_params
         )
 
-    # Can only be run by persistent tasks!
-    async def publish(self, dtype, message, backend_params={}):
-        # Convert dtype to a topic
-        topic = self.RUNTIME_DTYPES.replace("<dtype_label>", dtype.name)
-        await self._backend.publish(topic=topic, message=message, **backend_params)
-
-
-if __name__ == "__main__":
-
-    # PubSub client is a lightweight caller of pub sub backends.
-    # It also operates via queues. Subscriptions take a queue, and the pubsub
-    # client adds items to the queue as they arrive. It also supports multiple
-    # backends.
-    #
-    # Future: specifying pub/sub topics by backend.
-
-    pub_addr = "tcp://127.0.0.1:5000"
-    sub_addr = "tcp://127.0.0.1:5001"
+    async def unsubscribe_dtype(self, dtype: DataType):
+        if dtype not in self.subscriptions:
+            return
+        self.subscriptions.discard(dtype)
 
-    # Run ZMQ Broker in another process
+        await self._backend.unsubscribe(self.topic(dtype))
 
-    q: asyncio.Queue[TypedData] = asyncio.Queue()
+    async def publish(self, dtype: DataType, message, backend_params={}):
+        # Convert dtype to a topic
+        await self._backend.publish(
+            topic=self.topic(dtype), message=message, **backend_params
+        )
 
-    async def hello_world():
-        while True:
-            item = await q.get()
-            print(f"Hello World! I got: {item.data}")
+    async def close(self):
+        """Drop all subscriptions and close the backend.  Idempotent.
 
-    async def main():
+        The client owns its backend: one client per twin, torn down with it.
+        """
 
-        zmq_backend = ZMQ_PS_Client(pub_addr, sub_addr)
-        await zmq_backend.connect()
+        for dtype in list(self.subscriptions):
+            await self.unsubscribe_dtype(dtype)
 
-        ps_client = PubSubClient(zmq_backend)
+        await self._backend.close()
 
-        # Test PubSub
 
-        worker = asyncio.create_task(hello_world())
+async def connect_stream_client(
+    namespace: str, pub_addr: Optional[str] = None, sub_addr: Optional[str] = None
+) -> PubSubClient:
+    """Build and connect a namespaced stream client from configuration."""
 
-        await ps_client.subscribe_to_dtype(DataType("hello"), q)
+    pub_addr, sub_addr = stream_addresses(pub_addr, sub_addr)
 
-        for i in range(10):
-            await ps_client.publish(DataType("hello"), f"message {i}")
-            await asyncio.sleep(1)
+    backend = ZMQ_PS_Client(pub_addr, sub_addr)
+    await backend.connect()
 
-    asyncio.run(main())
+    return PubSubClient(backend, namespace)

From 655d1c6cf4b1921db00575a52233fae2033cc21b Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:37:00 +0200
Subject: [PATCH 02/19] Runtime lifecycle: terminal stop, failure state,
 describe()

M0.1/M0.3/M0.5 in the runtime.

- DTRuntime.stop(): terminal, idempotent, per-twin.  Cancels every task
  the runtime owns (component main loops, callbacks, dtype consumers,
  barrier loops), waits for them with a bound, abandons what does not
  settle, then drops the twin's subscriptions and closes its stream
  client.  The shared engine is never touched.  In-flight backend
  tasks are cancelled best-effort through the awaiting task and
  abandoned after the timeout.  start() after stop() raises.
- All done-callbacks are now the one cancellation-safe _task_done,
  which routes component exceptions into runtime state (failed + last
  error) instead of the loop's exception handler.  RuntimeAPI's own
  background-task set is gone: it uses the runtime's task plumbing.
- RuntimeAPI exposes the injected, namespaced, connected stream client
  as `runtime.stream` -- persistent components publish through it and
  never build transport clients or see addresses.
- describe() returns a serializable graph/state summary; print_graph()
  is now just a rendering of it.
- Ride-alongs: awaited the coroutine in _internal_agent_inference,
  fixed WindowDataType.__eq__ comparing a field to itself, removed the
  dead _to_block machinery and the redundant truthy_list.  Barrier's
  loop is owned by the runtime (Barrier.run) instead of an untracked
  task with a result-raising callback.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/components.py |  14 +-
 src/digitaltwin/runtime.py    | 303 ++++++++++++++++++++++++----------
 2 files changed, 225 insertions(+), 92 deletions(-)

diff --git a/src/digitaltwin/components.py b/src/digitaltwin/components.py
index d3c4b83..043f6b5 100644
--- a/src/digitaltwin/components.py
+++ b/src/digitaltwin/components.py
@@ -55,7 +55,7 @@ def __hash__(self):
     def __eq__(self, obj):
         return (
             isinstance(obj, WindowDataType)
-            and obj.dtype == obj.dtype
+            and obj.dtype == self.dtype
             and obj.name == self.name
         )
 
@@ -204,10 +204,6 @@ def add_dtype(self, dtype: DataType, hard: Optional[bool] = None):
         else:
             return WindowDataType(dtype, self.name)
 
-    async def start(self):
-        self.loop_task = asyncio.create_task(self._loop())
-        self.loop_task.add_done_callback(lambda r: r.result())
-
     async def put(self, in_data: TypedData):
         dtype = in_data.dtype
         if not (self.dtypes[dtype]):
@@ -251,7 +247,11 @@ async def get(self, dtype: DataType, wait=True):
         else:
             return self.output_queues[dtype].get_nowait()
 
-    async def _loop(self):
+    async def run(self):
+        """Barrier main loop.  Runs until cancelled -- the runtime owns the
+        task (see `DTRuntime.add_barrier`), so it is cancelled on stop and
+        its failures are routed into the runtime state."""
+
         # wait for there to be at least one task
         while self.count_soft + self.count_hard == 0:
             await asyncio.sleep(0.01)
@@ -336,7 +336,7 @@ async def main():
         b.add_dtype(orange)
         b.add_dtype(pear, hard=True)
 
-        await b.start()
+        asyncio.create_task(b.run())
 
         t1 = asyncio.create_task(apple_producer())
         t2 = asyncio.create_task(orange_producer())
diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index 2fe755f..70df551 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -1,16 +1,41 @@
+import asyncio
+import logging
+
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Iterable, cast
-
-from .components import *
-from .components import _TwinComponent
-from .streaming import *
+from enum import StrEnum
+from typing import Any, Callable, Optional
 
 from radical.asyncflow import WorkflowEngine  # type: ignore
-import logging
+
+from .components import (
+    NULL_DTYPE,
+    TRUTHY,
+    Barrier,
+    DataType,
+    ModelInvestigator,
+    SciAgent,
+    SplitTask,
+    TypedData,
+    UtilityTask,
+    _TwinComponent,
+)
+from .streaming import PubSubClient
 
 logger = logging.getLogger(__name__)
 
+# bounded wait for in-flight tasks to settle on stop()
+STOP_TIMEOUT = 10.0
+
+
+class RuntimeState(StrEnum):
+    """Lifecycle of a twin runtime.  `stopped` is terminal."""
+
+    READY = "ready"
+    RUNNING = "running"
+    STOPPED = "stopped"
+    FAILED = "failed"
+
 
 @dataclass
 class _AnnotatedComponent:
@@ -37,18 +62,30 @@ class _AnnotatedComponent:
     model_publish_cb = None
 
 
-class RuntimeAPI(ABC):
+class RuntimeAPI:
+    """What a twin component sees of its runtime."""
+
     ON_INPUT = "runtime/ON_INPUT"
     ON_OUTPUT = "runtime/ON_OUTPUT"
     ON_MODEL_PUBLISH = "runtime/ON_PUBLISH"
     ON_FILTERED_INPUT = "runtime/ON_FILTER_INPUT"
     ON_FILTERED_OUTPUT = "runtime/ON_FILTER_OUTPUT"
 
-    def __init__(self, ant: _AnnotatedComponent, agent_inf: Callable):
+    def __init__(self, runtime: "DTRuntime", ant: _AnnotatedComponent):
+        self._runtime = runtime
         self._ant = ant
         self._internal_add_investigator: Optional[Callable] = None
-        self._internal_agent_inference: Callable = agent_inf
-        self._background_tasks: set[asyncio.Task] = set()
+
+    @property
+    def stream(self) -> PubSubClient:
+        """The twin's namespaced, connected stream client.
+
+        Persistent components publish their output through it (the runtime
+        subscribes to that dtype and feeds the graph with it).  Components
+        never build their own transport clients and never see addresses.
+        """
+
+        return self._runtime.streamer
 
     def subscribe_to_topic(self, topic: str, task: Callable):
         self._ant.subscriptions[topic].append(task)
@@ -58,18 +95,12 @@ def publish_new_model(self, model_kwargs={}, acc_kwargs={}):
         self._ant.accuracy_kwargs = acc_kwargs
         self._ant.has_published_model.set()
         if self._ant.model_publish_cb is not None:
-            bk = asyncio.create_task(
-                self._ant.model_publish_cb(
-                    self._ant.component, model_kwargs, acc_kwargs
-                )
+            self._runtime._to_asyncio_task(
+                self._ant.model_publish_cb,
+                self._ant.component,
+                model_kwargs,
+                acc_kwargs,
             )
-            self._background_tasks.add(bk)
-
-            def done(r):
-                r.result()  # for error propagation
-                self._background_tasks.discard(r)
-
-            bk.add_done_callback(done)
 
     def set_inference_task(self, task: Callable):
         self._ant.inference_task = task
@@ -104,7 +135,7 @@ def update_model_selector(self, *args, **kwargs):
     async def get_inference(
         self, input_d: TypedData, output_dtype: DataType
     ) -> TypedData:
-        return await self._internal_agent_inference(input_d, output_dtype)
+        return await self._runtime._internal_agent_inference(input_d, output_dtype)
 
 
 class DTRuntime:
@@ -134,7 +165,6 @@ class DTRuntime:
 
     Conversions:
     _to_asyncio_task     ASYNC_DEF
-    _to_block            FLOW_BLOCK
     _call_await          resolves FLOW_BLOCK, FLOW_TASK, and ASYNC)
 
 
@@ -168,33 +198,100 @@ def __init__(self, flow: WorkflowEngine, streamer: PubSubClient):
         # list of barriers: order of PRODUCE --> CONSUME
         self.barriers: dict[DataType, list[Barrier]] = defaultdict(list)
 
-        self.truthy_list: list[_AnnotatedComponent] = []
-
         self.running_tasks: set[asyncio.Task] = set()
 
         self.is_start = asyncio.Event()
 
-        @flow.block
-        async def to_block(func, *args, **kwargs):
-            await func(*args, **kwargs)
-
-        self._to_block = to_block
+        self.state = RuntimeState.READY
+        self.last_error: Optional[str] = None
 
     def start(self):
+        if self.state is RuntimeState.STOPPED:
+            raise RuntimeError("stop() is terminal - this twin cannot be restarted")
+
+        if self.state is RuntimeState.READY:
+            self.state = RuntimeState.RUNNING
+
         self.is_start.set()
 
+    async def stop(self, timeout: float = STOP_TIMEOUT):
+        """Tear this twin down.  Terminal, idempotent, per-twin.
+
+        Cancels every task the runtime owns (component main loops,
+        callbacks, dtype consumers, barrier loops), then drops the twin's
+        stream subscriptions and closes its stream client.  The execution
+        engine is shared and never touched here.
+
+        In-flight backend tasks: cancelling the task that awaits one
+        propagates the cancellation into the backend call, which is a
+        best-effort cancel.  Whatever has not settled after `timeout` is
+        abandoned with a warning -- stop() never waits unboundedly.
+        """
+
+        if self.state is RuntimeState.STOPPED:
+            return
+
+        # also stops _to_asyncio_task from creating new work
+        self.state = RuntimeState.STOPPED
+
+        tasks, self.running_tasks = self.running_tasks, set()
+        for task in tasks:
+            task.cancel()
+
+        if tasks:
+            _, pending = await asyncio.wait(tasks, timeout=timeout)
+            if pending:
+                logger.warning(
+                    "abandoning %d task(s) which ignored cancellation: %s",
+                    len(pending),
+                    ", ".join(str(task.get_coro()) for task in pending),
+                )
+
+        try:
+            await asyncio.wait_for(self.streamer.close(), timeout)
+        except asyncio.TimeoutError:
+            logger.warning("stream client did not close within %ss", timeout)
+        except Exception as exc:
+            self._record_error(exc)
+
     async def _call_await(self, func, *args, **kwargs):
         await func(*args, **kwargs)
 
-    def _to_asyncio_task(self, func, *args, **kwargs):
-        result = asyncio.create_task(func(*args, **kwargs))
-        self.running_tasks.add(result)
+    def _to_asyncio_task(self, func, *args, **kwargs) -> Optional[asyncio.Task]:
+        if self.state is RuntimeState.STOPPED:
+            logger.debug("twin is stopped - not running %s", func)
+            return None
+
+        task = asyncio.create_task(func(*args, **kwargs))
+        self.running_tasks.add(task)
+        task.add_done_callback(self._task_done)
+
+        return task
+
+    def _task_done(self, task: asyncio.Task):
+        """Done callback for all runtime tasks: cancellation-safe, and it
+        routes component failures into the twin state instead of dumping
+        them into the event loop's exception handler."""
+
+        self.running_tasks.discard(task)
 
-        def done(r):
-            r.result()  # for error propagation
-            self.running_tasks.discard(r)
+        if task.cancelled():
+            return
+
+        exc = task.exception()
+        if exc is not None:
+            self._record_error(exc)
+
+    def _record_error(self, exc: BaseException):
+        self.last_error = f"{type(exc).__name__}: {exc}"
+        logger.error("twin component failed: %s", self.last_error, exc_info=exc)
 
-        result.add_done_callback(done)
+        # a stopped twin stays stopped, but keeps the error for inspection
+        if self.state is not RuntimeState.STOPPED:
+            self.state = RuntimeState.FAILED
+
+    def _api(self, ant: _AnnotatedComponent) -> RuntimeAPI:
+        return RuntimeAPI(self, ant)
 
     def add_task(
         self,
@@ -215,7 +312,6 @@ def add_task(
             true_data = TypedData(TRUTHY, True)
             # call as a block so it recieves Ctrl-C
             self._to_asyncio_task(self._run_component, ant_comp, true_data)
-            self.truthy_list.append(ant_comp)
 
     def add_investigator(
         self,
@@ -244,7 +340,7 @@ def add_investigator(
         self.components[input_dtype].append(ant_comp)
 
         # start up its main loop
-        rt = RuntimeAPI(ant_comp, self._internal_agent_inference)
+        rt = self._api(ant_comp)
         self._to_asyncio_task(investigator.main_loop, rt, *args, **kwargs)
 
         # is input_dtype TRUTHY? That doesn't make sense for investigators!
@@ -252,7 +348,7 @@ def add_investigator(
     def _internal_add_investigator(self, ant: _AnnotatedComponent):
         # subscribe to model publishes
         # start up its main loop
-        rt = RuntimeAPI(ant, self._internal_agent_inference)
+        rt = self._api(ant)
         self._to_asyncio_task(ant.component.main_loop, rt)
 
     def add_agent(
@@ -282,7 +378,7 @@ def add_agent(
         self.components[input_dtype].append(ant_comp)
 
         # start up its main loop. Agents get a patched start_investigator
-        rt = RuntimeAPI(ant_comp, self._internal_agent_inference)
+        rt = self._api(ant_comp)
         rt._internal_add_investigator = self._internal_add_investigator
         self._to_asyncio_task(agent.main_loop, rt, *args, **kwargs)
 
@@ -293,7 +389,9 @@ async def _internal_agent_inference(self, in_data: TypedData, req_dtype: DataTyp
             # call its selected model
             if component.output_dtype == req_dtype and component.is_persistent is False:
                 # check if agent or investigator
-                answer = self._run_component(component, in_data, skip_queue_out=True)
+                answer = await self._run_component(
+                    component, in_data, skip_queue_out=True
+                )
                 if answer is None:
                     return TypedData(NULL_DTYPE, None)
                 return answer
@@ -316,7 +414,7 @@ def add_barrier(self, barrier: Barrier):
                     self._barrier_consumer, dtype, self.barriers[dtype][-2], barrier
                 )
 
-        self._to_asyncio_task(barrier.start)
+        self._to_asyncio_task(barrier.run)
 
         # I need a consumer per dtype per barrier.
 
@@ -373,12 +471,12 @@ async def _run_component(
                 # else: output is null.
 
                 # run mainloop as async task
-                rt = RuntimeAPI(ant, self._internal_agent_inference)
+                rt = self._api(ant)
                 logger.info(f"Run {type(ant.component).__name__} main loop")
                 self._to_asyncio_task(ant.component.main_loop, rt, in_data)
                 return
 
-            rt = RuntimeAPI(ant, self._internal_agent_inference)
+            rt = self._api(ant)
             logger.info(f"Run {type(ant.component).__name__} main loop")
             answer = await ant.component.main_loop(rt, in_data)
 
@@ -531,44 +629,79 @@ async def _launch_consumer(self, dtype: DataType):
                 logger.info(f"Dequeue: {t_data.dtype}")
                 await self._dtype_consumer(t_data)
 
-    def print_graph(self):
-        print()
-        print("=" * 30)
-        out = "=" * 30 + "\n"
-        print("Digital Twin Flow: ")
-        out += "Digital Twin Flow: \n"
-
-        # start with TRUTHY
-        print("IN: (TRUTHY)")
-        out += "IN: (TRUTHY)\n"
-
-        for ant in self.truthy_list:
-            print(f"\t{ant.output_dtype}: {ant.component} ({ant.is_persistent})")
-            out += f"\t{ant.output_dtype}: {ant.component} ({ant.is_persistent})\n"
-
-        # rest of tasks
-        for input_dtype in self.components:
-            if input_dtype == TRUTHY:
-                continue
-            print(f"IN: {input_dtype}")
-            out += f"IN: {input_dtype}\n"
-            for ant in self.components[input_dtype]:
-                print(f"\t{ant.output_dtype}: {ant.component}  ({ant.is_persistent})")
-                out += f"\t{ant.output_dtype}: {ant.component}  ({ant.is_persistent})\n"
-
-        print("BARRIERS: ")
-        out += "BARRIERS: \n"
-        for dtype in self.barriers:
-            print(f"\t {dtype} --> ", end="")
-            out += f"\t {dtype} --> "
-            for barrier in self.barriers[dtype]:
-                is_hard = barrier.dtypes[dtype]
-                print(f"{barrier.name}{'' if is_hard else ']W'} --> ", end="")
-                out += f"{barrier.name}{'' if is_hard else ']W'} --> "
-            print()
-            out += "\n"
-
-        print("=" * 30)
-        out += "=" * 30
-        print()
+    def describe(self) -> dict:
+        """Serializable summary of the twin: graph, dtypes, state.
+
+        Introspection only, but this is the format that goes on the wire --
+        `print_graph()` is just a rendering of it.
+        """
+
+        def described(ant: _AnnotatedComponent) -> dict:
+            entry = {
+                "component": type(ant.component).__name__,
+                "input_dtype": ant.input_dtype.name,
+                "output_dtype": ant.output_dtype.name,
+                "is_persistent": ant.is_persistent,
+            }
+            if ant.investigators:
+                entry["investigators"] = [
+                    described(inv) for inv in ant.investigators.values()
+                ]
+            return entry
+
+        components = [
+            described(ant) for ants in self.components.values() for ant in ants
+        ]
+
+        return {
+            "namespace": self.streamer.namespace,
+            "state": str(self.state),
+            "last_error": self.last_error,
+            "components": components,
+            "dtypes": sorted(
+                {entry["input_dtype"] for entry in components}
+                | {entry["output_dtype"] for entry in components}
+            ),
+            # per dtype, the ordered chain of barriers it passes through
+            "barriers": {
+                dtype.name: [
+                    {"name": barrier.name, "hard": barrier.dtypes[dtype]}
+                    for barrier in chain
+                ]
+                for dtype, chain in self.barriers.items()
+            },
+        }
+
+    def print_graph(self) -> str:
+        """Human-readable rendering of `describe()`."""
+
+        info = self.describe()
+
+        by_input: dict[str, list[dict]] = defaultdict(list)
+        for entry in info["components"]:
+            by_input[entry["input_dtype"]].append(entry)
+
+        lines = ["=" * 30, f"Digital Twin Flow: {info['namespace']} [{info['state']}]"]
+
+        for input_dtype, entries in by_input.items():
+            lines.append(f"IN: {input_dtype}")
+            for entry in entries:
+                lines.append(
+                    f"\t{entry['output_dtype']}: {entry['component']}"
+                    f" ({entry['is_persistent']})"
+                )
+
+        lines.append("BARRIERS: ")
+        for dtype, chain in info["barriers"].items():
+            hops = "".join(
+                f"{barrier['name']}{'' if barrier['hard'] else ']W'} --> "
+                for barrier in chain
+            )
+            lines.append(f"\t {dtype} --> {hops}")
+
+        lines.append("=" * 30)
+
+        out = "\n".join(lines)
+        print(f"\n{out}\n")
+
         return out

From b2925f832931494d6baaab9cbb099d4ac09bff0f Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:37:14 +0200
Subject: [PATCH 03/19] Packaging: real dependencies, exports, pytest/tox
 scaffolding

M0.5/M0.6.  pyproject declared no dependencies at all; the versions
listed are the ones the framework is developed against (dist name
`rhapsody-py`).  `digitaltwin` now exports its public API, and pytest
(asyncio auto mode, tests under test/unit) plus tox are configured.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pyproject.toml              | 21 +++++++++++++++-
 src/digitaltwin/__init__.py | 49 +++++++++++++++++++++++++++++++++++++
 tox.ini                     |  8 ++++++
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 tox.ini

diff --git a/pyproject.toml b/pyproject.toml
index dd1f82b..8d9ae10 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,6 +5,25 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "digitaltwin"
 version = "0.0.1"
+description = "Experimental digital twin framework"
+requires-python = ">=3.11"
+dependencies = [
+    "radical.asyncflow>=0.5.0",
+    "rhapsody-py>=0.4.0",
+    "pyzmq>=25",
+    "cloudpickle>=3",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=8",
+    "pytest-asyncio>=0.24",
+]
 
 [tool.setuptools.packages.find]
-where = ["src"]
\ No newline at end of file
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["test/unit"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
diff --git a/src/digitaltwin/__init__.py b/src/digitaltwin/__init__.py
index e69de29..3253440 100644
--- a/src/digitaltwin/__init__.py
+++ b/src/digitaltwin/__init__.py
@@ -0,0 +1,49 @@
+"""Experimental digital twin framework."""
+
+from .components import (
+    NULL_DTYPE,
+    TRUTHY,
+    Barrier,
+    DataType,
+    ModelInvestigator,
+    SciAgent,
+    SplitTask,
+    TypedData,
+    UtilityTask,
+    WindowDataType,
+    WindowedTypeData,
+)
+from .config import stream_addresses
+from .runtime import DTRuntime, RuntimeAPI, RuntimeState
+from .streaming import (
+    PubSubBackend,
+    PubSubClient,
+    ZMQ_Broker,
+    ZMQ_BrokerProcess,
+    ZMQ_PS_Client,
+    connect_stream_client,
+)
+
+__all__ = [
+    "NULL_DTYPE",
+    "TRUTHY",
+    "Barrier",
+    "DTRuntime",
+    "DataType",
+    "ModelInvestigator",
+    "PubSubBackend",
+    "PubSubClient",
+    "RuntimeAPI",
+    "RuntimeState",
+    "SciAgent",
+    "SplitTask",
+    "TypedData",
+    "UtilityTask",
+    "WindowDataType",
+    "WindowedTypeData",
+    "ZMQ_Broker",
+    "ZMQ_BrokerProcess",
+    "ZMQ_PS_Client",
+    "connect_stream_client",
+    "stream_addresses",
+]
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..4f36c3e
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist = py311, py312, py313
+isolated_build = true
+skip_missing_interpreters = true
+
+[testenv]
+extras = test
+commands = pytest {posargs}

From 78f6357b95f4b82d1f7e41a4a18aafe58c056dcc Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:37:14 +0200
Subject: [PATCH 04/19] Unit tests for teardown, namespacing, streams and
 broker

M0.6.  Covers stop being terminal/idempotent, component failure ->
failed state + last error, bounded abandon of tasks which ignore
cancellation, and a leak assertion (no lingering tasks, subscriptions,
sockets or contexts after teardown); namespace isolation of identical
dtype labels on one broker; persistent components publishing through
the injected stream client end to end, including two twins side by
side; and broker start/stop cycling with random ports.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 test/unit/conftest.py            |  50 +++++++++
 test/unit/test_broker.py         |  56 ++++++++++
 test/unit/test_runtime_stop.py   | 182 +++++++++++++++++++++++++++++++
 test/unit/test_runtime_stream.py | 106 ++++++++++++++++++
 test/unit/test_streaming.py      |  93 ++++++++++++++++
 5 files changed, 487 insertions(+)
 create mode 100644 test/unit/conftest.py
 create mode 100644 test/unit/test_broker.py
 create mode 100644 test/unit/test_runtime_stop.py
 create mode 100644 test/unit/test_runtime_stream.py
 create mode 100644 test/unit/test_streaming.py

diff --git a/test/unit/conftest.py b/test/unit/conftest.py
new file mode 100644
index 0000000..a3cd2dc
--- /dev/null
+++ b/test/unit/conftest.py
@@ -0,0 +1,50 @@
+import asyncio
+
+import pytest
+
+from digitaltwin import ZMQ_BrokerProcess, connect_stream_client
+
+
+@pytest.fixture
+async def broker():
+    """An embedded stream broker on a random loopback port."""
+
+    proc = ZMQ_BrokerProcess()
+    await proc.start()
+    try:
+        yield proc
+    finally:
+        await proc.stop()
+
+
+@pytest.fixture
+async def stream_clients(broker):
+    """Factory for namespaced stream clients on the fixture broker.
+
+    All clients it hands out are closed when the test ends -- a client
+    left open would be caught by the leak assertions of the next test.
+    """
+
+    clients = []
+
+    async def make(namespace: str):
+        client = await connect_stream_client(namespace, *broker.get_connection_str())
+        clients.append(client)
+        return client
+
+    try:
+        yield make
+    finally:
+        for client in clients:
+            await client.close()
+
+
+@pytest.fixture
+async def no_task_leaks():
+    """Assert that the test leaves no asyncio task behind."""
+
+    before = asyncio.all_tasks()
+    yield
+    leaked = {task for task in asyncio.all_tasks() if task not in before}
+    leaked.discard(asyncio.current_task())
+    assert not leaked, f"leaked tasks: {leaked}"
diff --git a/test/unit/test_broker.py b/test/unit/test_broker.py
new file mode 100644
index 0000000..50d8449
--- /dev/null
+++ b/test/unit/test_broker.py
@@ -0,0 +1,56 @@
+"""M0.4 -- embedded broker: random ports, loopback default, clean stop."""
+
+from digitaltwin import ZMQ_BrokerProcess
+from digitaltwin.config import DEFAULT_BIND_HOST
+
+
+async def test_start_reports_bound_random_loopback_ports():
+    proc = ZMQ_BrokerProcess()
+    try:
+        pub_addr, sub_addr = await proc.start()
+
+        for addr in (pub_addr, sub_addr):
+            assert addr.startswith(f"tcp://{DEFAULT_BIND_HOST}:")
+            assert int(addr.rsplit(":", 1)[1]) > 0
+
+        assert pub_addr != sub_addr
+        assert proc.get_connection_str() == (pub_addr, sub_addr)
+        assert proc.is_alive()
+
+    finally:
+        await proc.stop()
+
+
+async def test_start_stop_cycling():
+    proc = ZMQ_BrokerProcess()
+    seen = []
+
+    for _ in range(3):
+        addrs = await proc.start()
+        assert proc.is_alive()
+        seen.append(addrs)
+
+        # start is idempotent while running
+        assert await proc.start() == addrs
+
+        await proc.stop()
+        assert not proc.is_alive()
+        assert proc.get_connection_str() == (None, None)
+
+        # stop is idempotent
+        await proc.stop()
+
+    # random ports: a fresh cycle does not reuse the previous bind
+    assert len(set(seen)) == len(seen)
+
+
+async def test_configured_addresses_are_honoured():
+    # port 0 is the POSIX 'pick one for me' spelling; either way the
+    # caller learns the actual port from the reported addresses
+    proc = ZMQ_BrokerProcess("tcp://127.0.0.1:0", "tcp://127.0.0.1:0")
+    try:
+        pub_addr, sub_addr = await proc.start()
+        assert not pub_addr.endswith(":0")
+        assert not sub_addr.endswith(":0")
+    finally:
+        await proc.stop()
diff --git a/test/unit/test_runtime_stop.py b/test/unit/test_runtime_stop.py
new file mode 100644
index 0000000..36ef7fd
--- /dev/null
+++ b/test/unit/test_runtime_stop.py
@@ -0,0 +1,182 @@
+"""M0.1 -- teardown: terminal/idempotent stop, failure routing, no leaks."""
+
+import asyncio
+
+import pytest
+
+from digitaltwin import (
+    NULL_DTYPE,
+    TRUTHY,
+    Barrier,
+    DTRuntime,
+    DataType,
+    RuntimeState,
+    UtilityTask,
+)
+
+TICK = DataType("tick")
+
+# DTRuntime never calls into the engine itself -- components do.  The unit
+# tests use components which do not submit any work, so no engine is needed.
+NO_FLOW = None
+
+
+class Forever(UtilityTask):
+    """A persistent component that never returns."""
+
+    async def main_loop(self, runtime, in_data):
+        while True:
+            await asyncio.sleep(0.01)
+
+
+class SlowToCancel(UtilityTask):
+    """A component that does not settle when cancelled -- stop() must
+    abandon it instead of waiting for it."""
+
+    def __init__(self, flow, release: asyncio.Event):
+        super().__init__(flow)
+        self.release = release
+
+    async def main_loop(self, runtime, in_data):
+        try:
+            await asyncio.sleep(3600)
+        except asyncio.CancelledError:
+            await self.release.wait()
+
+
+class Boom(UtilityTask):
+    async def main_loop(self, runtime, in_data):
+        raise ValueError("component exploded")
+
+
+async def test_stop_is_terminal_and_idempotent(stream_clients):
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+
+    assert runtime.state is RuntimeState.READY
+    runtime.start()
+    assert runtime.state is RuntimeState.RUNNING
+
+    await asyncio.sleep(0.1)
+    await runtime.stop()
+    assert runtime.state is RuntimeState.STOPPED
+
+    await runtime.stop()  # idempotent
+    assert runtime.state is RuntimeState.STOPPED
+
+    with pytest.raises(RuntimeError):
+        runtime.start()
+
+
+async def test_stop_cancels_tasks_and_tears_down_the_stream(
+    stream_clients, no_task_leaks
+):
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+
+    barrier = Barrier("b")
+    barrier.add_dtype(TICK)
+    runtime.add_barrier(barrier)
+
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    runtime.start()
+
+    # the persistent component subscribed the runtime to its output dtype
+    await asyncio.sleep(0.3)
+    assert runtime.running_tasks
+    assert TICK in client.subscriptions
+
+    await runtime.stop()
+
+    # no lingering tasks, subscriptions, sockets or contexts
+    assert runtime.running_tasks == set()
+    assert client.subscriptions == set()
+    assert client._backend.pub_soc is None
+    assert client._backend.sub_soc is None
+    assert client._backend._ctx.closed
+
+
+async def test_stop_abandons_tasks_that_ignore_cancellation(
+    stream_clients, no_task_leaks
+):
+    release = asyncio.Event()
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.add_task(SlowToCancel(NO_FLOW, release), TRUTHY, TICK, is_persistent=True)
+    runtime.start()
+    await asyncio.sleep(0.1)
+
+    # bounded: stop returns even though the component does not settle
+    await asyncio.wait_for(runtime.stop(timeout=0.2), timeout=2.0)
+    assert runtime.state is RuntimeState.STOPPED
+
+    # the abandoned task is no longer owned by the runtime
+    assert runtime.running_tasks == set()
+
+    release.set()
+    await asyncio.sleep(0.05)
+
+
+async def test_component_failure_sets_failed_state(stream_clients):
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.add_task(Boom(NO_FLOW), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    await asyncio.sleep(0.1)
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "ValueError: component exploded"
+
+    # stop still works on a failed twin, and keeps the error
+    await runtime.stop()
+    assert runtime.state is RuntimeState.STOPPED
+    assert runtime.last_error == "ValueError: component exploded"
+
+
+async def test_stopped_runtime_starts_no_new_work(stream_clients):
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.start()
+    await runtime.stop()
+
+    assert runtime._to_asyncio_task(asyncio.sleep, 0) is None
+    assert runtime.running_tasks == set()
+
+
+async def test_describe_is_serializable(stream_clients):
+    import json
+
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    inference = DataType("inference")
+
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    runtime.add_task(Forever(NO_FLOW), inference, NULL_DTYPE)
+
+    barrier = Barrier("b", hard=False)
+    barrier.add_dtype(TICK)
+    runtime.add_barrier(barrier)
+
+    info = runtime.describe()
+    assert json.loads(json.dumps(info)) == info
+
+    assert info["namespace"] == "twin-a"
+    assert info["state"] == "ready"
+    assert info["last_error"] is None
+    assert sorted(info["dtypes"]) == ["NULL", "TRUE", "inference", "tick"]
+    assert info["barriers"] == {"tick": [{"name": "b", "hard": False}]}
+    assert info["components"] == [
+        {
+            "component": "Forever",
+            "input_dtype": "TRUE",
+            "output_dtype": "tick",
+            "is_persistent": True,
+        },
+        {
+            "component": "Forever",
+            "input_dtype": "inference",
+            "output_dtype": "NULL",
+            "is_persistent": False,
+        },
+    ]
+
+    # print_graph is a rendering of describe()
+    assert "Forever" in runtime.print_graph()
+
+    await runtime.stop()
diff --git a/test/unit/test_runtime_stream.py b/test/unit/test_runtime_stream.py
new file mode 100644
index 0000000..8e09e5a
--- /dev/null
+++ b/test/unit/test_runtime_stream.py
@@ -0,0 +1,106 @@
+"""M0.3 -- persistent components publish through the injected stream client.
+
+No `function_task` wrapping, no hand-built ZMQ client, no address in sight:
+the component only ever touches `runtime.stream`.
+"""
+
+import asyncio
+
+from digitaltwin import (
+    NULL_DTYPE,
+    TRUTHY,
+    DTRuntime,
+    DataType,
+    ModelInvestigator,
+    TypedData,
+    UtilityTask,
+)
+
+SENSOR = DataType("sensor")
+INFERENCE = DataType("inference")
+
+
+class Sensor(UtilityTask):
+    async def main_loop(self, runtime, in_data):
+        value = 0
+        while True:
+            await runtime.stream.publish(SENSOR, value)
+            value += 1
+            await asyncio.sleep(0.05)
+
+
+class Doubler(ModelInvestigator):
+    async def main_loop(self, runtime):
+        async def inference(in_data, **model_kwargs):
+            return TypedData(INFERENCE, in_data.data * 2)
+
+        runtime.set_inference_task(inference)
+        runtime.publish_new_model()
+
+
+class Sink(UtilityTask):
+    def __init__(self, flow, received):
+        super().__init__(flow)
+        self.received = received
+
+    async def main_loop(self, runtime, in_data):
+        self.received.append(in_data.data)
+
+
+async def test_persistent_component_publishes_through_runtime_stream(
+    stream_clients, no_task_leaks
+):
+    received: list[int] = []
+
+    runtime = DTRuntime(None, await stream_clients("twin-a"))
+    runtime.add_task(Sensor(None), TRUTHY, SENSOR, is_persistent=True)
+    runtime.add_investigator(Doubler(None), SENSOR, INFERENCE)
+    runtime.add_task(Sink(None, received), INFERENCE, NULL_DTYPE)
+
+    runtime.start()
+
+    async def wait_for_data():
+        while len(received) < 3:
+            await asyncio.sleep(0.05)
+
+    await asyncio.wait_for(wait_for_data(), timeout=10.0)
+    await runtime.stop()
+
+    # the sensor counts up, the investigator doubles
+    assert received[:3] == [received[0], received[0] + 2, received[0] + 4]
+    assert all(value % 2 == 0 for value in received)
+
+
+async def test_two_twins_on_one_broker_stay_separate(stream_clients, no_task_leaks):
+    """Same component code, same dtype labels, two namespaces: the twins
+    must not see each other's data."""
+
+    received_a: list[int] = []
+    received_b: list[int] = []
+
+    twins = []
+    for namespace, received in (("twin-a", received_a), ("twin-b", received_b)):
+        runtime = DTRuntime(None, await stream_clients(namespace))
+        runtime.add_task(Sensor(None), TRUTHY, SENSOR, is_persistent=True)
+        runtime.add_task(Sink(None, received), SENSOR, NULL_DTYPE)
+        runtime.start()
+        twins.append(runtime)
+
+    async def wait_for_data():
+        while len(received_a) < 5 or len(received_b) < 5:
+            await asyncio.sleep(0.05)
+
+    try:
+        await asyncio.wait_for(wait_for_data(), timeout=10.0)
+    finally:
+        for runtime in twins:
+            await runtime.stop()
+
+    # each twin sees exactly its own monotonic sensor sequence -- crosstalk
+    # would show up as duplicated values
+    for received in (received_a, received_b):
+        assert received == sorted(set(received))
+
+    # tearing one twin down does not disturb its sibling
+    assert twins[0].last_error is None
+    assert twins[1].last_error is None
diff --git a/test/unit/test_streaming.py b/test/unit/test_streaming.py
new file mode 100644
index 0000000..75b362d
--- /dev/null
+++ b/test/unit/test_streaming.py
@@ -0,0 +1,93 @@
+"""M0.2 -- topic namespacing and stream client teardown."""
+
+import asyncio
+
+import pytest
+
+from digitaltwin import DataType, PubSubClient
+
+SENSOR = DataType("sensor")
+
+
+async def _drain(queue: asyncio.Queue, timeout=5.0):
+    return await asyncio.wait_for(queue.get(), timeout)
+
+
+def test_topics_are_namespaced_and_terminated():
+    a = PubSubClient(backend=None, namespace="twin-a")
+    b = PubSubClient(backend=None, namespace="twin-b")
+
+    assert a.topic(SENSOR) == "dt/twin-a/dtypes/sensor\x00"
+    assert a.topic(SENSOR) != b.topic(SENSOR)
+
+    # the terminator keeps a label from prefix-matching a longer one
+    assert not a.topic(DataType("sen")).startswith(a.topic(SENSOR)[:-1])
+    assert not a.topic(SENSOR).startswith(a.topic(DataType("sen")))
+
+
+async def test_identical_dtype_labels_do_not_cross_subscribe(stream_clients):
+    """The multi-tenancy correctness target: same dtype label, two
+    namespaces, one broker -- no crosstalk."""
+
+    twin_a = await stream_clients("twin-a")
+    twin_b = await stream_clients("twin-b")
+
+    queue_a: asyncio.Queue = asyncio.Queue()
+    queue_b: asyncio.Queue = asyncio.Queue()
+
+    await twin_a.subscribe_to_dtype(SENSOR, queue_a)
+    await twin_b.subscribe_to_dtype(SENSOR, queue_b)
+    await asyncio.sleep(0.2)  # let the subscriptions reach the broker
+
+    for i in range(3):
+        await twin_a.publish(SENSOR, f"a-{i}")
+
+    received = await _drain(queue_a)
+    assert received.dtype == SENSOR
+    assert received.data == "a-0"
+
+    await asyncio.sleep(0.2)
+    assert queue_b.empty()
+
+
+async def test_unsubscribe_dtype_stops_delivery(stream_clients):
+    twin = await stream_clients("twin-a")
+    queue: asyncio.Queue = asyncio.Queue()
+
+    await twin.subscribe_to_dtype(SENSOR, queue)
+    await asyncio.sleep(0.2)
+
+    await twin.publish(SENSOR, "first")
+    assert (await _drain(queue)).data == "first"
+
+    await twin.unsubscribe_dtype(SENSOR)
+    assert SENSOR not in twin.subscriptions
+    await asyncio.sleep(0.2)
+
+    await twin.publish(SENSOR, "second")
+    await asyncio.sleep(0.2)
+    assert queue.empty()
+
+    # re-subscription is possible after unsubscribe
+    await twin.subscribe_to_dtype(SENSOR, queue)
+
+
+async def test_close_releases_task_sockets_and_context(broker, no_task_leaks):
+    from digitaltwin import connect_stream_client
+
+    twin = await connect_stream_client("twin-a", *broker.get_connection_str())
+    backend = twin._backend
+
+    await twin.subscribe_to_dtype(SENSOR, asyncio.Queue())
+
+    await twin.close()
+
+    assert twin.subscriptions == set()
+    assert backend.topics == {}
+    assert backend.pub_soc is None and backend.sub_soc is None
+    assert backend._ctx.closed
+
+    # idempotent, and a closed client refuses further use
+    await twin.close()
+    with pytest.raises(RuntimeError):
+        await twin.publish(SENSOR, "nope")

From 34ffe756379b947c3dd625243f273143713d2181 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:51:02 +0200
Subject: [PATCH 05/19] Migrate the demos to the M0 API

M0.3/M0.6.  Persistent components are plain async main_loops which
publish through `runtime.stream`: the `@flow.function_task` wrapper and
the hand-built `ZMQ_PS_Client` inside every sensor are gone, and with
them the last address literals in demo code (a function task would have
been cloudpickled to a backend slot for the lifetime of the twin).
Every demo now opens one namespaced stream client via
`connect_stream_client()` and tears its twin down with `runtime.stop()`
before shutting the shared engine down.

`local_broker.py` and the demos resolve the broker addresses through
`digitaltwin.config`, so the two-terminal model still works and the
broker is configurable; the READMEs document the resolution and the
loopback binding policy.

Not migrated: test/rose_streaming (untracked in this repo).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 README.md                                     |  30 +++-
 src/digitaltwin/remote/README.md              |   2 +-
 test/00-start-stop/run_me.py                  |  11 +-
 test/01-start-inference-stop/dtypes.py        |   3 -
 test/01-start-inference-stop/run_me.py        |   9 +-
 test/01-start-inference-stop/sensor.py        |  23 +--
 test/02-start-redeploy-stop/dtypes.py         |   3 -
 test/02-start-redeploy-stop/run_me.py         |   9 +-
 test/02-start-redeploy-stop/sensor.py         |  21 +--
 test/03-conditional-redeploy/dtypes.py        |   3 -
 test/03-conditional-redeploy/run_me.py        |   9 +-
 test/03-conditional-redeploy/sensor.py        |  21 +--
 test/04-start-agent-stop/dtypes.py            |   3 -
 test/04-start-agent-stop/run_me.py            |   9 +-
 test/04-start-agent-stop/sensor.py            |  21 +--
 test/05-agent-w-multi-investigators/dtypes.py |   3 -
 test/05-agent-w-multi-investigators/run_me.py |   9 +-
 test/05-agent-w-multi-investigators/sensor.py |  21 +--
 test/06-agent-pi/dtypes.py                    |   3 -
 test/06-agent-pi/in_timer.py                  |  19 +--
 test/06-agent-pi/run_me.py                    |   9 +-
 test/07-barrier/run_me.py                     |  12 +-
 test/07-barrier/sensor.py                     |  24 +--
 test/09-remote/dtypes.py                      |   3 -
 test/09-remote/run_me.py                      |   4 +-
 test/09-remote/sensor.py                      |  21 +--
 test/demo/01-start-inference-stop/dtypes.py   |   3 -
 test/demo/01-start-inference-stop/run_me.py   |   9 +-
 test/demo/01-start-inference-stop/sensor.py   |  29 +---
 test/demo/02-ex-situ/dtypes.py                |   3 -
 test/demo/02-ex-situ/run_me.py                |   9 +-
 test/demo/02-ex-situ/sensor.py                |  29 +---
 .../dtypes.py                                 |   3 -
 .../run_me.py                                 |   9 +-
 .../sensor.py                                 | 157 ++++++++----------
 test/demo/03-mnst-active-learn/dtypes.py      |   3 -
 test/demo/03-mnst-active-learn/run_me.py      |   9 +-
 test/demo/03-mnst-active-learn/sensor.py      |  75 ++++-----
 test/demo/04-start-agent-stop/dtypes.py       |   3 -
 test/demo/04-start-agent-stop/run_me.py       |   9 +-
 test/demo/04-start-agent-stop/sensor.py       |  29 +---
 test/demo/05-multi-investigators/dtypes.py    |   3 -
 test/demo/05-multi-investigators/in_timer.py  |  29 +---
 test/demo/05-multi-investigators/run_me.py    |   9 +-
 test/demo/06-multi-agent/dtypes.py            |   3 -
 test/demo/06-multi-agent/run_me.py            |   9 +-
 test/demo/06-multi-agent/sensor.py            |  59 +++----
 test/local_broker.py                          |  18 +-
 test/remote_service.py                        |  14 +-
 test/rose_agent/run_me.py                     |  34 +---
 test/rose_example/run_me.py                   |  33 +---
 51 files changed, 323 insertions(+), 572 deletions(-)

diff --git a/README.md b/README.md
index 5f51efc..163bde8 100644
--- a/README.md
+++ b/README.md
@@ -23,11 +23,29 @@ Not yet implemented:
 - - Docs
 - - Type Annotation 
 
-## Running tests:
+## Running the unit tests:
 
-1. `pip install -e .`
-2. `cd tests/`
-3. In one terminal, run `local_broker.py`
-4. In second terminal, cd into the test and run `run_me.py`
+1. `pip install .[test]`
+2. `pytest` (or `tox` for all supported interpreters)
 
-**When running tests: be sure to start the ZMQ PubSub broker!**
+The unit tests start their own stream broker on a random port; no setup.
+
+## Running the demos:
+
+1. `pip install .`
+2. `cd test/`
+3. In one terminal, run `local_broker.py` -- it prints the addresses it
+   bound
+4. In a second terminal, cd into the demo and run `run_me.py`
+
+**When running a demo: be sure to start the ZMQ PubSub broker!**
+
+Both sides resolve the broker addresses the same way: `DT_STREAM_PUB_ADDR`
+and `DT_STREAM_SUB_ADDR`, defaulting to `tcp://127.0.0.1:5000` and `:5001`
+(see `digitaltwin.config`).  Set them in both terminals to move the broker.
+
+**Binding policy**: the broker binds to loopback by default, and it must
+stay that way unless you know what you are doing -- pubsub payloads are
+cloudpickled, so anyone who can reach the broker ports can execute code in
+every subscriber.  A non-loopback bind needs an explicit configuration and
+a private/firewalled network.
diff --git a/src/digitaltwin/remote/README.md b/src/digitaltwin/remote/README.md
index c8b4075..053ad97 100644
--- a/src/digitaltwin/remote/README.md
+++ b/src/digitaltwin/remote/README.md
@@ -50,7 +50,7 @@ The service unpickles the arguments, invokes the corresponding method on the wra
 1. **Install the project** (including the new `cloudpickle` dependency):
 
    ```bash
-   pip install -e .
+   pip install .
    ```
 
 2. **Launch the remote service** from your application or a terminal:
diff --git a/test/00-start-stop/run_me.py b/test/00-start-stop/run_me.py
index 0e8c1c7..5323ce2 100644
--- a/test/00-start-stop/run_me.py
+++ b/test/00-start-stop/run_me.py
@@ -6,7 +6,7 @@
 
 from digitaltwin.components import *
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 
 from radical.asyncflow.logging import init_default_logger
 
@@ -20,9 +20,6 @@
 
 # Globals:
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
-
 
 if __name__ == "__main__":
 
@@ -34,9 +31,8 @@ async def main():
         exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
         flow = await WorkflowEngine.create(backend=exe)
 
-        stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-        await stream_backend.connect()
-        pubsub_client = PubSubClient(stream_backend)
+        # create the twin's namespaced stream client
+        pubsub_client = await connect_stream_client("00-start-stop")
 
         runtime = DTRuntime(flow, pubsub_client)
 
@@ -46,6 +42,7 @@ async def main():
         # let it run....
         print("Sleeping...")
         await asyncio.sleep(10)
+        await runtime.stop()
         await flow.shutdown()
 
     asyncio.run(main())
diff --git a/test/01-start-inference-stop/dtypes.py b/test/01-start-inference-stop/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/01-start-inference-stop/dtypes.py
+++ b/test/01-start-inference-stop/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/01-start-inference-stop/run_me.py b/test/01-start-inference-stop/run_me.py
index bc15185..ebecb1e 100644
--- a/test/01-start-inference-stop/run_me.py
+++ b/test/01-start-inference-stop/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("01-start-inference-stop")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -51,6 +49,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(30)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/01-start-inference-stop/sensor.py b/test/01-start-inference-stop/sensor.py
index e6e854d..52ffe9c 100644
--- a/test/01-start-inference-stop/sensor.py
+++ b/test/01-start-inference-stop/sensor.py
@@ -4,7 +4,6 @@
 import time
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -19,20 +18,10 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(30):
-                # val = random.random()
-                val = time.monotonic_ns()
-                print(f"Sensor val: {val} - {i}")
-                await pclient.publish(SENSOR_DTYPE, val)
-                await asyncio.sleep(0.5)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        for i in range(30):
+            # val = random.random()
+            val = time.monotonic_ns()
+            print(f"Sensor val: {val} - {i}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
+            await asyncio.sleep(0.5)
diff --git a/test/02-start-redeploy-stop/dtypes.py b/test/02-start-redeploy-stop/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/02-start-redeploy-stop/dtypes.py
+++ b/test/02-start-redeploy-stop/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/02-start-redeploy-stop/run_me.py b/test/02-start-redeploy-stop/run_me.py
index a776eb8..bcad8ff 100644
--- a/test/02-start-redeploy-stop/run_me.py
+++ b/test/02-start-redeploy-stop/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("02-start-redeploy-stop")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/02-start-redeploy-stop/sensor.py b/test/02-start-redeploy-stop/sensor.py
index d9ef7be..74189a7 100644
--- a/test/02-start-redeploy-stop/sensor.py
+++ b/test/02-start-redeploy-stop/sensor.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,19 +17,9 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            while True:
-                await asyncio.sleep(1)
-                val = random.random()
-                print(f"Sensor val: {val}")
-                await pclient.publish(SENSOR_DTYPE, val)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        while True:
+            await asyncio.sleep(1)
+            val = random.random()
+            print(f"Sensor val: {val}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
diff --git a/test/03-conditional-redeploy/dtypes.py b/test/03-conditional-redeploy/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/03-conditional-redeploy/dtypes.py
+++ b/test/03-conditional-redeploy/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/03-conditional-redeploy/run_me.py b/test/03-conditional-redeploy/run_me.py
index 9e7a29b..74378bd 100644
--- a/test/03-conditional-redeploy/run_me.py
+++ b/test/03-conditional-redeploy/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("03-conditional-redeploy")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE =================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/03-conditional-redeploy/sensor.py b/test/03-conditional-redeploy/sensor.py
index d9ef7be..74189a7 100644
--- a/test/03-conditional-redeploy/sensor.py
+++ b/test/03-conditional-redeploy/sensor.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,19 +17,9 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            while True:
-                await asyncio.sleep(1)
-                val = random.random()
-                print(f"Sensor val: {val}")
-                await pclient.publish(SENSOR_DTYPE, val)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        while True:
+            await asyncio.sleep(1)
+            val = random.random()
+            print(f"Sensor val: {val}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
diff --git a/test/04-start-agent-stop/dtypes.py b/test/04-start-agent-stop/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/04-start-agent-stop/dtypes.py
+++ b/test/04-start-agent-stop/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/04-start-agent-stop/run_me.py b/test/04-start-agent-stop/run_me.py
index 050bfec..836b0e1 100644
--- a/test/04-start-agent-stop/run_me.py
+++ b/test/04-start-agent-stop/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("04-start-agent-stop")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/04-start-agent-stop/sensor.py b/test/04-start-agent-stop/sensor.py
index d9ef7be..74189a7 100644
--- a/test/04-start-agent-stop/sensor.py
+++ b/test/04-start-agent-stop/sensor.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,19 +17,9 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            while True:
-                await asyncio.sleep(1)
-                val = random.random()
-                print(f"Sensor val: {val}")
-                await pclient.publish(SENSOR_DTYPE, val)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        while True:
+            await asyncio.sleep(1)
+            val = random.random()
+            print(f"Sensor val: {val}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
diff --git a/test/05-agent-w-multi-investigators/dtypes.py b/test/05-agent-w-multi-investigators/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/05-agent-w-multi-investigators/dtypes.py
+++ b/test/05-agent-w-multi-investigators/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/05-agent-w-multi-investigators/run_me.py b/test/05-agent-w-multi-investigators/run_me.py
index 050bfec..cdfdace 100644
--- a/test/05-agent-w-multi-investigators/run_me.py
+++ b/test/05-agent-w-multi-investigators/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("05-agent-w-multi-investigators")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/05-agent-w-multi-investigators/sensor.py b/test/05-agent-w-multi-investigators/sensor.py
index d9ef7be..74189a7 100644
--- a/test/05-agent-w-multi-investigators/sensor.py
+++ b/test/05-agent-w-multi-investigators/sensor.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,19 +17,9 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            while True:
-                await asyncio.sleep(1)
-                val = random.random()
-                print(f"Sensor val: {val}")
-                await pclient.publish(SENSOR_DTYPE, val)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        while True:
+            await asyncio.sleep(1)
+            val = random.random()
+            print(f"Sensor val: {val}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
diff --git a/test/06-agent-pi/dtypes.py b/test/06-agent-pi/dtypes.py
index 1a59cad..449c16e 100644
--- a/test/06-agent-pi/dtypes.py
+++ b/test/06-agent-pi/dtypes.py
@@ -2,6 +2,3 @@
 
 TIMER_TRIGGER_DTYPE = DataType("timer")
 PI_DTYPE = DataType("pi value")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/06-agent-pi/in_timer.py b/test/06-agent-pi/in_timer.py
index 71f4e26..99f75d6 100644
--- a/test/06-agent-pi/in_timer.py
+++ b/test/06-agent-pi/in_timer.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,18 +17,8 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            counter = 0
-            while True:
-                await pclient.publish(TIMER_TRIGGER_DTYPE, counter)
-                await asyncio.sleep(1)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        counter = 0
+        while True:
+            await runtime.stream.publish(TIMER_TRIGGER_DTYPE, counter)
+            await asyncio.sleep(1)
diff --git a/test/06-agent-pi/run_me.py b/test/06-agent-pi/run_me.py
index eba4d66..a7a4688 100644
--- a/test/06-agent-pi/run_me.py
+++ b/test/06-agent-pi/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("06-agent-pi")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(60)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/07-barrier/run_me.py b/test/07-barrier/run_me.py
index 8fd188b..a7bd5fc 100644
--- a/test/07-barrier/run_me.py
+++ b/test/07-barrier/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE, Barrier, DataType
 
 from sensor import MySensor
@@ -14,9 +14,6 @@
 from radical.asyncflow.logging import init_default_logger
 import logging
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
-
 logger = logging.getLogger(__name__)
 
 # put it all together
@@ -32,10 +29,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("07-barrier")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -97,6 +92,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(30)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/07-barrier/sensor.py b/test/07-barrier/sensor.py
index ff02df5..22ecdf2 100644
--- a/test/07-barrier/sensor.py
+++ b/test/07-barrier/sensor.py
@@ -4,12 +4,8 @@
 import time
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import DataType, UtilityTask
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
-
 import random
 
 import logging
@@ -21,19 +17,11 @@ class MySensor(UtilityTask):
     def __init__(self, flow: WorkflowEngine, delay, output_dt: DataType):
         super().__init__(flow)
         self.flow = flow
-
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(30):
-                print(f"Publish {output_dt}. Val: {i}")
-                await pclient.publish(output_dt, i)
-                await asyncio.sleep(delay)
-
-        self.task = task
+        self.delay = delay
+        self.output_dt = output_dt
 
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        for i in range(30):
+            print(f"Publish {self.output_dt}. Val: {i}")
+            await runtime.stream.publish(self.output_dt, i)
+            await asyncio.sleep(self.delay)
diff --git a/test/09-remote/dtypes.py b/test/09-remote/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/09-remote/dtypes.py
+++ b/test/09-remote/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/09-remote/run_me.py b/test/09-remote/run_me.py
index 6ad0685..35ac51f 100644
--- a/test/09-remote/run_me.py
+++ b/test/09-remote/run_me.py
@@ -1,3 +1,4 @@
+import os
 import time
 
 
@@ -33,7 +34,8 @@ def main():
     logging.getLogger("radical.asyncflow").setLevel(logging.WARNING)
     logging.getLogger("rhapsody").setLevel(logging.WARNING)
 
-    remote = RemoteDTOrchestrator(address="tcp://127.0.0.1:5555")
+    address = os.environ.get("DT_REMOTE_SERVICE_ADDR", "tcp://127.0.0.1:5555")
+    remote = RemoteDTOrchestrator(address=address)
 
     runtime = remote.new_session()
 
diff --git a/test/09-remote/sensor.py b/test/09-remote/sensor.py
index 2f652cf..833d3f7 100644
--- a/test/09-remote/sensor.py
+++ b/test/09-remote/sensor.py
@@ -3,7 +3,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -18,19 +17,9 @@ def __init__(self, flow: WorkflowEngine, *args, **kwargs):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(10):
-                await asyncio.sleep(1)
-                val = random.random()
-                print(f"Sensor val: {val}")
-                await pclient.publish(SENSOR_DTYPE, val)
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+        for i in range(10):
+            await asyncio.sleep(1)
+            val = random.random()
+            print(f"Sensor val: {val}")
+            await runtime.stream.publish(SENSOR_DTYPE, val)
diff --git a/test/demo/01-start-inference-stop/dtypes.py b/test/demo/01-start-inference-stop/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/demo/01-start-inference-stop/dtypes.py
+++ b/test/demo/01-start-inference-stop/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/01-start-inference-stop/run_me.py b/test/demo/01-start-inference-stop/run_me.py
index f93a7ad..0b1c7bd 100644
--- a/test/demo/01-start-inference-stop/run_me.py
+++ b/test/demo/01-start-inference-stop/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("01-start-inference-stop")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -51,6 +49,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(65)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/01-start-inference-stop/sensor.py b/test/demo/01-start-inference-stop/sensor.py
index 1d4c9b2..a3e3623 100644
--- a/test/demo/01-start-inference-stop/sensor.py
+++ b/test/demo/01-start-inference-stop/sensor.py
@@ -5,7 +5,6 @@
 import time
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -21,25 +20,15 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(30):
-                f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
-                await pclient.publish(SENSOR_DTYPE, i)
-                f.flush()
-                await asyncio.sleep(2)
+    async def main_loop(self, runtime, in_data):
 
-            f.close()
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
 
-        self.task = task
+        for i in range(30):
+            f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
+            await runtime.stream.publish(SENSOR_DTYPE, i)
+            f.flush()
+            await asyncio.sleep(2)
 
-    async def main_loop(self, runtime, in_data):
-        await self.task()
+        f.close()
diff --git a/test/demo/02-ex-situ/dtypes.py b/test/demo/02-ex-situ/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/demo/02-ex-situ/dtypes.py
+++ b/test/demo/02-ex-situ/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/02-ex-situ/run_me.py b/test/demo/02-ex-situ/run_me.py
index 23ba269..f5ad421 100644
--- a/test/demo/02-ex-situ/run_me.py
+++ b/test/demo/02-ex-situ/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("02-ex-situ")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -51,6 +49,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(70)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/02-ex-situ/sensor.py b/test/demo/02-ex-situ/sensor.py
index 1d4c9b2..a3e3623 100644
--- a/test/demo/02-ex-situ/sensor.py
+++ b/test/demo/02-ex-situ/sensor.py
@@ -5,7 +5,6 @@
 import time
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -21,25 +20,15 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(30):
-                f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
-                await pclient.publish(SENSOR_DTYPE, i)
-                f.flush()
-                await asyncio.sleep(2)
+    async def main_loop(self, runtime, in_data):
 
-            f.close()
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
 
-        self.task = task
+        for i in range(30):
+            f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
+            await runtime.stream.publish(SENSOR_DTYPE, i)
+            f.flush()
+            await asyncio.sleep(2)
 
-    async def main_loop(self, runtime, in_data):
-        await self.task()
+        f.close()
diff --git a/test/demo/03-mnst-active-learn-multi-stream/dtypes.py b/test/demo/03-mnst-active-learn-multi-stream/dtypes.py
index 66bf647..42c61da 100644
--- a/test/demo/03-mnst-active-learn-multi-stream/dtypes.py
+++ b/test/demo/03-mnst-active-learn-multi-stream/dtypes.py
@@ -4,6 +4,3 @@
 FASHION_CAMERA_DTYPE = DataType("fashion")
 DIGIT_DTYPE = DataType("digit-class")
 FASHION_DTYPE = DataType("fashion-class")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/03-mnst-active-learn-multi-stream/run_me.py b/test/demo/03-mnst-active-learn-multi-stream/run_me.py
index 7ba9ce4..079a5a3 100644
--- a/test/demo/03-mnst-active-learn-multi-stream/run_me.py
+++ b/test/demo/03-mnst-active-learn-multi-stream/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -32,10 +32,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("03-mnst-active-learn-multi-stream")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -59,6 +57,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(70)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/03-mnst-active-learn-multi-stream/sensor.py b/test/demo/03-mnst-active-learn-multi-stream/sensor.py
index 92967dd..dbac8d7 100644
--- a/test/demo/03-mnst-active-learn-multi-stream/sensor.py
+++ b/test/demo/03-mnst-active-learn-multi-stream/sensor.py
@@ -6,7 +6,6 @@
 
 import numpy as np
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 
@@ -30,51 +29,41 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("number.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            # Load the dataset
-            _, (test_images, test_labels) = mnist.load_data()
-
-            # Normalize the images to values between 0 and 1
-            test_images = test_images / 255.0
-
-            # Convert labels to one-hot encoded format
-
-            target_labels = []
-            for num in range(10):
-                # start slowly....
-                target_labels.append(num)
-                mask = np.isin(test_labels, target_labels)
-                target = test_images[mask]
-                indices = np.random.choice(len(target), size=5)
-                for i in range(4):
-                    img = target[indices][i]
-                    label = test_labels[mask][indices][i]
-                    plt.figure()
-                    plot_image(None, label, img)
-                    plt.savefig("number.png")
-                    # request inference of image
-                    f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
-                    await pclient.publish(
-                        NUMBER_CAMERA_DTYPE, {"label": label, "img": img}
-                    )
-                    f.flush()
-                    await asyncio.sleep(2)
-
-            f.close()
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+
+        f = open("number.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
+
+        # Load the dataset
+        _, (test_images, test_labels) = mnist.load_data()
+
+        # Normalize the images to values between 0 and 1
+        test_images = test_images / 255.0
+
+        # Convert labels to one-hot encoded format
+
+        target_labels = []
+        for num in range(10):
+            # start slowly....
+            target_labels.append(num)
+            mask = np.isin(test_labels, target_labels)
+            target = test_images[mask]
+            indices = np.random.choice(len(target), size=5)
+            for i in range(4):
+                img = target[indices][i]
+                label = test_labels[mask][indices][i]
+                plt.figure()
+                plot_image(None, label, img)
+                plt.savefig("number.png")
+                # request inference of image
+                f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
+                await runtime.stream.publish(
+                    NUMBER_CAMERA_DTYPE, {"label": label, "img": img}
+                )
+                f.flush()
+                await asyncio.sleep(2)
+
+        f.close()
 
 
 class FashionCamera(UtilityTask):
@@ -82,48 +71,38 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("fashion.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            # Load the dataset
-            _, (test_images, test_labels) = fashion_mnist.load_data()
-
-            # Normalize the images to values between 0 and 1
-            test_images = test_images / 255.0
-
-            # Convert labels to one-hot encoded format
-
-            target_labels = []
-            for num in range(10):
-                # start slowly....
-                target_labels.append(num)
-                mask = np.isin(test_labels, target_labels)
-                target = test_images[mask]
-                indices = np.random.choice(len(target), size=5)
-                for i in range(4):
-                    img = target[indices][i]
-                    label = test_labels[mask][indices][i]
-                    plt.figure()
-                    plot_image(None, label, img)
-                    plt.savefig("fashion.png")
-                    # request inference of image
-                    f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
-                    await pclient.publish(
-                        FASHION_CAMERA_DTYPE, {"label": label, "img": img}
-                    )
-                    f.flush()
-                    await asyncio.sleep(2)
-
-            f.close()
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+
+        f = open("fashion.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
+
+        # Load the dataset
+        _, (test_images, test_labels) = fashion_mnist.load_data()
+
+        # Normalize the images to values between 0 and 1
+        test_images = test_images / 255.0
+
+        # Convert labels to one-hot encoded format
+
+        target_labels = []
+        for num in range(10):
+            # start slowly....
+            target_labels.append(num)
+            mask = np.isin(test_labels, target_labels)
+            target = test_images[mask]
+            indices = np.random.choice(len(target), size=5)
+            for i in range(4):
+                img = target[indices][i]
+                label = test_labels[mask][indices][i]
+                plt.figure()
+                plot_image(None, label, img)
+                plt.savefig("fashion.png")
+                # request inference of image
+                f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
+                await runtime.stream.publish(
+                    FASHION_CAMERA_DTYPE, {"label": label, "img": img}
+                )
+                f.flush()
+                await asyncio.sleep(2)
+
+        f.close()
diff --git a/test/demo/03-mnst-active-learn/dtypes.py b/test/demo/03-mnst-active-learn/dtypes.py
index 46003ee..a9172fb 100644
--- a/test/demo/03-mnst-active-learn/dtypes.py
+++ b/test/demo/03-mnst-active-learn/dtypes.py
@@ -2,6 +2,3 @@
 
 CAMERA_DTYPE = DataType("camera")
 DIGIT_DTYPE = DataType("digit-class")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/03-mnst-active-learn/run_me.py b/test/demo/03-mnst-active-learn/run_me.py
index 3cd516f..f3e7c10 100644
--- a/test/demo/03-mnst-active-learn/run_me.py
+++ b/test/demo/03-mnst-active-learn/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("03-mnst-active-learn")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -51,6 +49,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(85)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/03-mnst-active-learn/sensor.py b/test/demo/03-mnst-active-learn/sensor.py
index 518211d..37a5a3b 100644
--- a/test/demo/03-mnst-active-learn/sensor.py
+++ b/test/demo/03-mnst-active-learn/sensor.py
@@ -6,7 +6,6 @@
 
 import numpy as np
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 from al.sim import plot_image
@@ -28,46 +27,36 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            # Load the dataset
-            _, (test_images, test_labels) = mnist.load_data()
-
-            # Normalize the images to values between 0 and 1
-            test_images = test_images / 255.0
-
-            # Convert labels to one-hot encoded format
-
-            target_labels = []
-            for num in range(10):
-                # start slowly....
-                target_labels.append(num)
-                mask = np.isin(test_labels, target_labels)
-                target = test_images[mask]
-                indices = np.random.choice(len(target), size=5)
-                for i in range(4):
-                    img = target[indices][i]
-                    label = test_labels[mask][indices][i]
-                    plt.figure()
-                    plot_image(None, label, img)
-                    plt.savefig("sensor.png")
-                    # request inference of image
-                    f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
-                    await pclient.publish(CAMERA_DTYPE, {"label": label, "img": img})
-                    f.flush()
-                    await asyncio.sleep(2)
-
-            f.close()
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        await self.task()
+
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
+
+        # Load the dataset
+        _, (test_images, test_labels) = mnist.load_data()
+
+        # Normalize the images to values between 0 and 1
+        test_images = test_images / 255.0
+
+        # Convert labels to one-hot encoded format
+
+        target_labels = []
+        for num in range(10):
+            # start slowly....
+            target_labels.append(num)
+            mask = np.isin(test_labels, target_labels)
+            target = test_images[mask]
+            indices = np.random.choice(len(target), size=5)
+            for i in range(4):
+                img = target[indices][i]
+                label = test_labels[mask][indices][i]
+                plt.figure()
+                plot_image(None, label, img)
+                plt.savefig("sensor.png")
+                # request inference of image
+                f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
+                await runtime.stream.publish(CAMERA_DTYPE, {"label": label, "img": img})
+                f.flush()
+                await asyncio.sleep(2)
+
+        f.close()
diff --git a/test/demo/04-start-agent-stop/dtypes.py b/test/demo/04-start-agent-stop/dtypes.py
index 15f0e69..ca16e88 100644
--- a/test/demo/04-start-agent-stop/dtypes.py
+++ b/test/demo/04-start-agent-stop/dtypes.py
@@ -2,6 +2,3 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/04-start-agent-stop/run_me.py b/test/demo/04-start-agent-stop/run_me.py
index 050bfec..836b0e1 100644
--- a/test/demo/04-start-agent-stop/run_me.py
+++ b/test/demo/04-start-agent-stop/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("04-start-agent-stop")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/04-start-agent-stop/sensor.py b/test/demo/04-start-agent-stop/sensor.py
index a34827b..9eb2296 100644
--- a/test/demo/04-start-agent-stop/sensor.py
+++ b/test/demo/04-start-agent-stop/sensor.py
@@ -5,7 +5,6 @@
 import time
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -21,26 +20,16 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            for i in range(30):
-                f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
-                f.flush()
+    async def main_loop(self, runtime, in_data):
 
-                await pclient.publish(SENSOR_DTYPE, i)
-                await asyncio.sleep(1)
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
 
-            f.close()
+        for i in range(30):
+            f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
+            f.flush()
 
-        self.task = task
+            await runtime.stream.publish(SENSOR_DTYPE, i)
+            await asyncio.sleep(1)
 
-    async def main_loop(self, runtime, in_data):
-        await self.task()
+        f.close()
diff --git a/test/demo/05-multi-investigators/dtypes.py b/test/demo/05-multi-investigators/dtypes.py
index 1a59cad..449c16e 100644
--- a/test/demo/05-multi-investigators/dtypes.py
+++ b/test/demo/05-multi-investigators/dtypes.py
@@ -2,6 +2,3 @@
 
 TIMER_TRIGGER_DTYPE = DataType("timer")
 PI_DTYPE = DataType("pi value")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/05-multi-investigators/in_timer.py b/test/demo/05-multi-investigators/in_timer.py
index 8723ec9..9d08d8a 100644
--- a/test/demo/05-multi-investigators/in_timer.py
+++ b/test/demo/05-multi-investigators/in_timer.py
@@ -4,7 +4,6 @@
 import sys
 
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 import random
@@ -19,24 +18,14 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task
-        async def task():
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            for i in range(30):
-                f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
-                await pclient.publish(TIMER_TRIGGER_DTYPE, i)
-                f.flush()
-                await asyncio.sleep(1)
-
-            f.close()
+    async def main_loop(self, runtime, in_data):
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
 
-        self.task = task
+        for i in range(30):
+            f.write(f"[{datetime.datetime.now()}] Publish: {i} \n")
+            await runtime.stream.publish(TIMER_TRIGGER_DTYPE, i)
+            f.flush()
+            await asyncio.sleep(1)
 
-    async def main_loop(self, runtime, in_data):
-        await self.task()
+        f.close()
diff --git a/test/demo/05-multi-investigators/run_me.py b/test/demo/05-multi-investigators/run_me.py
index e23dd84..b4a8411 100644
--- a/test/demo/05-multi-investigators/run_me.py
+++ b/test/demo/05-multi-investigators/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -30,10 +30,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("05-multi-investigators")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -52,6 +50,7 @@ async def main():
     # let it run
     await asyncio.sleep(30)
     print("DONE======================")
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/06-multi-agent/dtypes.py b/test/demo/06-multi-agent/dtypes.py
index 589b947..376ed3f 100644
--- a/test/demo/06-multi-agent/dtypes.py
+++ b/test/demo/06-multi-agent/dtypes.py
@@ -3,6 +3,3 @@
 CAMERA_DTYPE = DataType("camera")
 DIGIT_DTYPE = DataType("digit-class")
 ENGLISH_DTYPE = DataType("English")
-
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
diff --git a/test/demo/06-multi-agent/run_me.py b/test/demo/06-multi-agent/run_me.py
index 7810fae..a8d8503 100644
--- a/test/demo/06-multi-agent/run_me.py
+++ b/test/demo/06-multi-agent/run_me.py
@@ -4,7 +4,7 @@
 from rhapsody.backends import ConcurrentExecutionBackend
 
 from digitaltwin.runtime import DTRuntime
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 from digitaltwin.components import TRUTHY, NULL_DTYPE
 
 from dtypes import *
@@ -32,10 +32,8 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client
+    pubsub_client = await connect_stream_client("06-multi-agent")
 
     runtime = DTRuntime(flow, pubsub_client)
 
@@ -57,6 +55,7 @@ async def main():
 
     # let it run
     await asyncio.sleep(85)
+    await runtime.stop()
     await flow.shutdown()
 
 
diff --git a/test/demo/06-multi-agent/sensor.py b/test/demo/06-multi-agent/sensor.py
index 0731b8c..44c3881 100644
--- a/test/demo/06-multi-agent/sensor.py
+++ b/test/demo/06-multi-agent/sensor.py
@@ -6,7 +6,6 @@
 
 import numpy as np
 from radical.asyncflow import WorkflowEngine
-from digitaltwin.streaming import ZMQ_PS_Client, PubSubClient
 from digitaltwin.components import UtilityTask
 from dtypes import *
 
@@ -24,47 +23,37 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        @self.flow.function_task(service=True)
-        async def task():
-
-            f = open("sensor.out", "w")
-            f.write("SENSOR MEASUREMENTS ========================= \n")
-
-            ps_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await ps_backend.connect()
-            pclient = PubSubClient(ps_backend)
-
-            # Load the dataset
-            _, (test_images, test_labels) = mnist.load_data()
+    async def main_loop(self, runtime, in_data):
 
-            # Normalize the images to values between 0 and 1
-            test_images = test_images / 255.0
+        f = open("sensor.out", "w")
+        f.write("SENSOR MEASUREMENTS ========================= \n")
 
-            # Convert labels to one-hot encoded format
+        # Load the dataset
+        _, (test_images, test_labels) = mnist.load_data()
 
-            target_labels = []
-            for num in range(10):
-                # start slowly....
-                target_labels.append(num)
-                mask = np.isin(test_labels, target_labels)
-                target = test_images[mask]
-                indices = np.random.choice(len(target), size=5)
-                for i in range(5):
-                    img = target[indices][i]
-                    label = test_labels[mask][indices][i]
+        # Normalize the images to values between 0 and 1
+        test_images = test_images / 255.0
 
-                    # request inference of image
-                    f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
-                    await pclient.publish(CAMERA_DTYPE, {"label": label, "img": img})
-                    f.flush()
-                    await asyncio.sleep(2)
+        # Convert labels to one-hot encoded format
 
-            f.close()
+        target_labels = []
+        for num in range(10):
+            # start slowly....
+            target_labels.append(num)
+            mask = np.isin(test_labels, target_labels)
+            target = test_images[mask]
+            indices = np.random.choice(len(target), size=5)
+            for i in range(5):
+                img = target[indices][i]
+                label = test_labels[mask][indices][i]
 
-        self.task = task
+                # request inference of image
+                f.write(f"[{datetime.datetime.now()}] Emit an image of {label} \n")
+                await runtime.stream.publish(CAMERA_DTYPE, {"label": label, "img": img})
+                f.flush()
+                await asyncio.sleep(2)
 
-    async def main_loop(self, runtime, in_data):
-        await self.task()
+        f.close()
 
 
 # Load the dataset
diff --git a/test/local_broker.py b/test/local_broker.py
index 9abe929..0af1a6f 100644
--- a/test/local_broker.py
+++ b/test/local_broker.py
@@ -1,9 +1,17 @@
+"""Standalone stream broker for the two-terminal demos.
+
+Addresses come from configuration (`DT_STREAM_PUB_ADDR` /
+`DT_STREAM_SUB_ADDR`, loopback defaults) -- the same resolution the
+demos use, so both terminals agree without any literal in the code.
+"""
+
+from digitaltwin.config import stream_addresses
 from digitaltwin.streaming import ZMQ_Broker
 
-pub_addr = "tcp://127.0.0.1:5000"
-sub_addr = "tcp://127.0.0.1:5001"
+if __name__ == "__main__":
+    broker = ZMQ_Broker(*stream_addresses())
 
-zq = ZMQ_Broker(pub_addr, sub_addr)
+    publish_addr, subscribe_addr = broker.bind()
+    print(f"stream broker: publish to {publish_addr}, subscribe on {subscribe_addr}")
 
-print(zq.get_connection_str())
-zq.run()
+    broker.run()
diff --git a/test/remote_service.py b/test/remote_service.py
index f608cba..2570856 100644
--- a/test/remote_service.py
+++ b/test/remote_service.py
@@ -2,19 +2,18 @@
 from concurrent.futures import ProcessPoolExecutor
 from radical.asyncflow import WorkflowEngine
 from rhapsody.backends import ConcurrentExecutionBackend
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 
 from digitaltwin.remote.remote_service import RemoteDTService
 
 from radical.asyncflow.logging import init_default_logger
 import logging
+import os
 
 logger = logging.getLogger(__name__)
 
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
-SERVICE = "tcp://127.0.0.1:5555"
+SERVICE = os.environ.get("DT_REMOTE_SERVICE_ADDR", "tcp://127.0.0.1:5555")
 
 # put it all together
 # sensor --> model --> data_sink
@@ -29,10 +28,9 @@ async def main():
     exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
     flow = await WorkflowEngine.create(backend=exe)
 
-    # create pubsub backend client
-    stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-    await stream_backend.connect()
-    pubsub_client = PubSubClient(stream_backend)
+    # create the twin's namespaced stream client (this prototype hosts a
+    # single session, so a single namespace will do)
+    pubsub_client = await connect_stream_client("remote-service")
 
     service = RemoteDTService(flow, SERVICE, pubsub_client)
     logger.info(f"Starting RemoteDTService on {SERVICE}")
diff --git a/test/rose_agent/run_me.py b/test/rose_agent/run_me.py
index df203a2..c09cc5b 100644
--- a/test/rose_agent/run_me.py
+++ b/test/rose_agent/run_me.py
@@ -13,7 +13,7 @@
 
 from digitaltwin.components import *
 from digitaltwin.runtime import DTRuntime, RuntimeAPI
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 
 from radical.asyncflow.logging import init_default_logger
 
@@ -24,8 +24,6 @@
 
 # Globals:
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
 
 # = LocalBackend()
 # Create the data types
@@ -208,27 +206,12 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        # register task
-        @self.flow.function_task
-        async def task():
-            logger.debug("Running Persistent Task inside AF task")
-            # open a stream handler on the task itself.
-            stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await stream_backend.connect()
-
-            pub_client = PubSubClient(stream_backend)
-
-            output_dtype = SENSOR_DTYPE
-            for i in range(100):
-                await asyncio.sleep(1)
-                logger.debug(f"Publish message with dtype: {output_dtype}")
-                await pub_client.publish(output_dtype, message="Hello!")
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        # start the task
-        await self.task()
+        output_dtype = SENSOR_DTYPE
+        for i in range(100):
+            await asyncio.sleep(1)
+            logger.debug(f"Publish message with dtype: {output_dtype}")
+            await runtime.stream.publish(output_dtype, message="Hello!")
 
 
 if __name__ == "__main__":
@@ -242,9 +225,7 @@ async def main():
         exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
         flow = await WorkflowEngine.create(backend=exe)
 
-        stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-        await stream_backend.connect()
-        pubsub_client = PubSubClient(stream_backend)
+        pubsub_client = await connect_stream_client("rose-agent")
 
         runtime = DTRuntime(flow, pubsub_client)
 
@@ -260,6 +241,7 @@ async def main():
 
         # let it run....
         await asyncio.sleep(4)
+        await runtime.stop()
         await flow.shutdown()
 
     asyncio.run(main())
diff --git a/test/rose_example/run_me.py b/test/rose_example/run_me.py
index db6d7eb..4ef84b3 100644
--- a/test/rose_example/run_me.py
+++ b/test/rose_example/run_me.py
@@ -12,7 +12,7 @@
 
 from digitaltwin.components import *
 from digitaltwin.runtime import DTRuntime, RuntimeAPI
-from digitaltwin.streaming import PubSubClient, ZMQ_PS_Client
+from digitaltwin.streaming import connect_stream_client
 
 from radical.asyncflow.logging import init_default_logger
 
@@ -23,8 +23,6 @@
 
 # Globals:
 
-ZMQ_PS_BROKER_PUB = "tcp://127.0.0.1:5000"
-ZMQ_PS_BROKER_SUB = "tcp://127.0.0.1:5001"
 
 # = LocalBackend()
 # Create the data types
@@ -97,27 +95,12 @@ def __init__(self, flow: WorkflowEngine):
         super().__init__(flow)
         self.flow = flow
 
-        # register task
-        @self.flow.function_task
-        async def task():
-            logger.debug("Running Persistent Task inside AF task")
-            # open a stream handler on the task itself.
-            stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB)
-            await stream_backend.connect()
-
-            pub_client = PubSubClient(stream_backend)
-
-            output_dtype = SENSOR_DTYPE
-            while True:
-                await asyncio.sleep(1)
-                logger.debug(f"Publish message with dtype: {output_dtype}")
-                await pub_client.publish(output_dtype, message="Hello!")
-
-        self.task = task
-
     async def main_loop(self, runtime, in_data):
-        # start the task
-        await self.task()
+        output_dtype = SENSOR_DTYPE
+        while True:
+            await asyncio.sleep(1)
+            logger.debug(f"Publish message with dtype: {output_dtype}")
+            await runtime.stream.publish(output_dtype, message="Hello!")
 
 
 if __name__ == "__main__":
@@ -127,9 +110,7 @@ async def main():
         exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
         flow = await WorkflowEngine.create(backend=exe)
 
-        stream_backend = ZMQ_PS_Client(ZMQ_PS_BROKER_PUB, ZMQ_PS_BROKER_SUB)
-        await stream_backend.connect()
-        pubsub_client = PubSubClient(stream_backend)
+        pubsub_client = await connect_stream_client("rose-example")
 
         runtime = DTRuntime(flow, pubsub_client)
 

From f57123ba4ebb1bc89380e816f009170e335e46eb Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 15:54:56 +0200
Subject: [PATCH 06/19] Harden the broker launcher stop path

Guard against stopping a process whose start() failed, and reformat.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/streaming.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index b2c2746..9c649e9 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -132,7 +132,10 @@ class ZMQ_BrokerProcess:
     def __init__(
         self, publish_addr: Optional[str] = None, subscribe_addr: Optional[str] = None
     ):
-        self._addrs = (publish_addr or RANDOM_PUB_ADDR, subscribe_addr or RANDOM_SUB_ADDR)
+        self._addrs = (
+            publish_addr or RANDOM_PUB_ADDR,
+            subscribe_addr or RANDOM_SUB_ADDR,
+        )
         self._proc: Optional[multiprocessing.process.BaseProcess] = None
 
         self.publish_addr: Optional[str] = None
@@ -192,14 +195,15 @@ def _start(self, timeout):
     def _stop(self, timeout):
         proc, self._proc = self._proc, None
 
-        if proc.is_alive():
-            proc.terminate()
-            proc.join(timeout)
+        if proc.pid is not None:  # None if start() itself failed
+            if proc.is_alive():
+                proc.terminate()
+                proc.join(timeout)
 
-        if proc.is_alive():
-            logger.warning("stream broker ignored terminate -- killing")
-            proc.kill()
-            proc.join(timeout)
+            if proc.is_alive():
+                logger.warning("stream broker ignored terminate -- killing")
+                proc.kill()
+                proc.join(timeout)
 
         proc.close()
 

From 1dbddfa25b9a23b78e521140f9f714b059843e2a Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:13:38 +0200
Subject: [PATCH 07/19] Review findings: stream resilience and join-safe
 teardown

Fable review of M0.  Stability fixes, no new surface:

- The receive loop no longer dies on a bad payload or a raising
  subscriber: malformed messages are dropped and logged, and a failing
  subscriber does not starve its siblings.  Any other exit of that loop
  is reported through a new `PubSubBackend.on_error` hook which the
  runtime routes into `_record_error` -- a silently stalled stream is
  now a `failed` twin with a last error, not silence.
- `stop()` memoizes its teardown: concurrent and repeated callers await
  the same task and thus only return once the twin is really down
  (M1's `twin_close` idempotency for free).  The caller's cancellation
  does not abort it -- stop is terminal.
- `start()` on a failed twin raises, like `start()` after `stop()`.
- Graph mutation after stop raises instead of registering components
  whose tasks are then silently dropped.
- `ZMQ_BrokerProcess` serializes start/stop with a lock: concurrent
  starts can no longer spawn a second broker or observe (None, None).
- `PubSubClient` rejects namespaces containing the topic separator or
  the terminator -- the aliasing the namespace exists to prevent.
- `ZMQ_PS_Client.close()` closes sockets and terminates the context in
  a `finally`: a cancelled close can no longer leak them permanently
  behind the one-shot `_closed` guard.
- Dropped the instance assignment in `PubSubBackend.__init__` which
  shadowed each subclass's `label`, and the unexplained sleep in
  `connect()` (the monitor already confirms the connection).
- Broker cycling test no longer asserts distinct ephemeral ports
  (nondeterministic); it asserts valid ports and that a broker from a
  fresh cycle actually proxies.  `local_broker` prints with flush.

New tests: malformed payload / failing subscriber survival, receive
loop exit reporting, stream failure -> failed twin, concurrent stop,
stopped-graph mutation, namespace validation.  21 passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/runtime.py     |  36 ++++++++--
 src/digitaltwin/streaming.py   | 127 +++++++++++++++++++++++----------
 test/local_broker.py           |   5 +-
 test/unit/test_broker.py       |  36 +++++++---
 test/unit/test_runtime_stop.py |  47 ++++++++++++
 test/unit/test_streaming.py    |  57 +++++++++++++++
 6 files changed, 256 insertions(+), 52 deletions(-)

diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index 70df551..60ab0ea 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -205,18 +205,29 @@ def __init__(self, flow: WorkflowEngine, streamer: PubSubClient):
         self.state = RuntimeState.READY
         self.last_error: Optional[str] = None
 
+        self._stop_task: Optional[asyncio.Task] = None
+
+        # a stalled stream is a twin failure, not a log line
+        streamer.on_error = self._record_error
+
     def start(self):
         if self.state is RuntimeState.STOPPED:
             raise RuntimeError("stop() is terminal - this twin cannot be restarted")
 
-        if self.state is RuntimeState.READY:
-            self.state = RuntimeState.RUNNING
+        if self.state is RuntimeState.FAILED:
+            raise RuntimeError(
+                f"twin has failed and cannot be started: {self.last_error}"
+            )
 
+        self.state = RuntimeState.RUNNING
         self.is_start.set()
 
     async def stop(self, timeout: float = STOP_TIMEOUT):
         """Tear this twin down.  Terminal, idempotent, per-twin.
 
+        Idempotent by joining: concurrent and repeated calls await the one
+        teardown, so `stop()` only ever returns once the twin is down.
+
         Cancels every task the runtime owns (component main loops,
         callbacks, dtype consumers, barrier loops), then drops the twin's
         stream subscriptions and closes its stream client.  The execution
@@ -228,12 +239,16 @@ async def stop(self, timeout: float = STOP_TIMEOUT):
         abandoned with a warning -- stop() never waits unboundedly.
         """
 
-        if self.state is RuntimeState.STOPPED:
-            return
+        if self._stop_task is None:
+            # flip the state before scheduling: no new work from here on
+            self.state = RuntimeState.STOPPED
+            self._stop_task = asyncio.ensure_future(self._teardown(timeout))
 
-        # also stops _to_asyncio_task from creating new work
-        self.state = RuntimeState.STOPPED
+        # every caller joins the one teardown; a cancelled caller does not
+        # abort it (stop is terminal)
+        await asyncio.shield(self._stop_task)
 
+    async def _teardown(self, timeout: float):
         tasks, self.running_tasks = self.running_tasks, set()
         for task in tasks:
             task.cancel()
@@ -293,6 +308,10 @@ def _record_error(self, exc: BaseException):
     def _api(self, ant: _AnnotatedComponent) -> RuntimeAPI:
         return RuntimeAPI(self, ant)
 
+    def _check_mutable(self):
+        if self.state is RuntimeState.STOPPED:
+            raise RuntimeError("twin is stopped - its graph cannot be changed")
+
     def add_task(
         self,
         task: UtilityTask,
@@ -300,6 +319,7 @@ def add_task(
         output_dtype: DataType,
         is_persistent=False,
     ):
+        self._check_mutable()
 
         ant_comp = _AnnotatedComponent(task, input_dtype, output_dtype, is_persistent)
 
@@ -321,6 +341,7 @@ def add_investigator(
         *args,
         **kwargs,
     ):
+        self._check_mutable()
         assert input_dtype != TRUTHY
 
         # check: is there already an investigator or agent assigned to this
@@ -359,6 +380,7 @@ def add_agent(
         *args,
         **kwargs,
     ):
+        self._check_mutable()
         assert input_dtype != TRUTHY
 
         # check: is there already an investigator or agent assigned to this
@@ -404,6 +426,8 @@ async def add_split_task(
 
     # add a barrier
     def add_barrier(self, barrier: Barrier):
+        self._check_mutable()
+
         # a barrier spans across multiple dtypes.... add in the order that
         # follows.
         for dtype in barrier.dtypes:
diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index 9c649e9..7ae5445 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -30,7 +30,16 @@ class PubSubBackend(ABC):
     label = "generic"
 
     def __init__(self):
-        self.label = "PubSubBackend"
+        # asynchronous failures (a dead receive loop above all) are reported
+        # here -- see PubSubClient.on_error.  A silently stalled stream is
+        # the failure mode this exists to prevent.
+        self.on_error: Optional[Callable[[BaseException], None]] = None
+
+    def _report_error(self, exc: BaseException):
+        logger.error("stream backend failed: %s", exc, exc_info=exc)
+
+        if self.on_error is not None:
+            self.on_error(exc)
 
     @abstractmethod
     async def connect(self, *args, **kwargs):
@@ -138,30 +147,36 @@ def __init__(
         )
         self._proc: Optional[multiprocessing.process.BaseProcess] = None
 
+        # serializes start/stop: concurrent starts must not spawn two
+        # brokers, and must not observe a half-started one
+        self._lock = asyncio.Lock()
+
         self.publish_addr: Optional[str] = None
         self.subscribe_addr: Optional[str] = None
 
     async def start(self, timeout: float = BROKER_START_TIMEOUT) -> tuple[str, str]:
         """Spawn the broker and return its bound (publish, subscribe) pair."""
 
-        if self._proc is not None:
-            return self.get_connection_str()
+        async with self._lock:
+            if self._proc is not None:
+                return self.get_connection_str()
 
-        # spawn + pipe read are blocking: keep them off the event loop
-        addrs = await asyncio.to_thread(self._start, timeout)
-        self.publish_addr, self.subscribe_addr = addrs
-        logger.info("stream broker at %s / %s", *addrs)
+            # spawn + pipe read are blocking: keep them off the event loop
+            addrs = await asyncio.to_thread(self._start, timeout)
+            self.publish_addr, self.subscribe_addr = addrs
+            logger.info("stream broker at %s / %s", *addrs)
 
-        return addrs
+            return addrs
 
     async def stop(self, timeout: float = BROKER_STOP_TIMEOUT):
         """Terminate the broker subprocess.  Idempotent."""
 
-        if self._proc is None:
-            return
+        async with self._lock:
+            if self._proc is None:
+                return
 
-        await asyncio.to_thread(self._stop, timeout)
-        self.publish_addr = self.subscribe_addr = None
+            await asyncio.to_thread(self._stop, timeout)
+            self.publish_addr = self.subscribe_addr = None
 
     def is_alive(self) -> bool:
         return self._proc is not None and self._proc.is_alive()
@@ -264,8 +279,10 @@ async def connect(self):
         if self.pub_soc is not None:
             await self._connect_socket(self.pub_soc, self.pub_addr)
 
-        self._task = asyncio.create_task(self._run())
-        await asyncio.sleep(0.1)
+        if self.sub_soc is not None:
+            self._task = asyncio.create_task(self._run())
+            self._task.add_done_callback(self._run_done)
+
         self.is_running.set()
 
     async def publish(self, topic, message):
@@ -312,40 +329,61 @@ async def close(self):
         self._closed = True
 
         task, self._task = self._task, None
-        if task is not None:
-            task.cancel()
-            with contextlib.suppress(asyncio.CancelledError):
-                await task
-
-        self.topics.clear()
+        try:
+            if task is not None:
+                task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await task
+        finally:
+            # the guard above makes close() a one-shot, so the sockets and
+            # the context have to go even if the await was cancelled
+            self.topics.clear()
 
-        for sock in (self.pub_soc, self.sub_soc):
-            if sock is not None:
-                sock.close(linger=0)
-        self.pub_soc = self.sub_soc = None
+            for sock in (self.pub_soc, self.sub_soc):
+                if sock is not None:
+                    sock.close(linger=0)
+            self.pub_soc = self.sub_soc = None
 
-        # all sockets are closed, so this returns immediately
-        self._ctx.term()
-        self.is_running.clear()
+            # all sockets are closed, so this returns immediately
+            self._ctx.term()
+            self.is_running.clear()
 
     def _check_open(self):
         if self._closed:
             raise RuntimeError("stream client is closed")
 
     async def _run(self):
-        if self.sub_soc is None:
-            return
-        try:
-            while True:
-                topic, message = await self.sub_soc.recv_multipart()
+        """Receive loop.  A single bad payload or a raising callback must
+        not take the stream down -- it is dropped and logged."""
+
+        while True:
+            frames = await self.sub_soc.recv_multipart()
+
+            try:
+                topic, message = frames
                 item = topic.decode("utf-8")
                 data = cloudpickle.loads(message)
-                for task in self.topics.get(item, []):
+            except Exception:
+                logger.exception("dropping malformed message: %r", frames[:1])
+                continue
+
+            for task in self.topics.get(item, []):
+                # one failing subscriber must not starve its siblings
+                # (CancelledError is not an Exception: close() still wins)
+                try:
                     await task(data)
-        except asyncio.CancelledError:
-            raise
-        except zmq.ContextTerminated:
-            pass
+                except Exception:
+                    logger.exception("subscriber failed on topic %r", item)
+
+    def _run_done(self, task: asyncio.Task):
+        """The receive loop only ends on close().  Any other exit means the
+        twin stopped receiving -- report it instead of stalling silently."""
+
+        if self._closed or task.cancelled():
+            return
+
+        exc = task.exception() or RuntimeError("stream receive loop exited")
+        self._report_error(exc)
 
 
 # The pubsub client abstracts away the specifics of the pub / sub
@@ -371,12 +409,27 @@ class PubSubClient:
     # For now, only support one backend. Future TODO: Add support for multiple backends
 
     def __init__(self, backend: PubSubBackend, namespace: str):
+        # a namespace carrying a separator would let two twins alias each
+        # other's topics -- the one thing the namespace exists to prevent
+        if not namespace or "/" in namespace or self.TOPIC_TERMINATOR in namespace:
+            raise ValueError(f"invalid stream namespace: {namespace!r}")
+
         self._backend = backend
         self.namespace = namespace
 
         # so I don't repeat
         self.subscriptions: set[DataType] = set()
 
+    @property
+    def on_error(self) -> Optional[Callable[[BaseException], None]]:
+        """Hook for asynchronous stream failures (see `PubSubBackend`)."""
+
+        return self._backend.on_error
+
+    @on_error.setter
+    def on_error(self, callback: Optional[Callable[[BaseException], None]]):
+        self._backend.on_error = callback
+
     def topic(self, dtype: DataType) -> str:
         return f"dt/{self.namespace}/dtypes/{dtype.name}{self.TOPIC_TERMINATOR}"
 
diff --git a/test/local_broker.py b/test/local_broker.py
index 0af1a6f..8587ffe 100644
--- a/test/local_broker.py
+++ b/test/local_broker.py
@@ -12,6 +12,9 @@
     broker = ZMQ_Broker(*stream_addresses())
 
     publish_addr, subscribe_addr = broker.bind()
-    print(f"stream broker: publish to {publish_addr}, subscribe on {subscribe_addr}")
+    print(
+        f"stream broker: publish to {publish_addr}, subscribe on {subscribe_addr}",
+        flush=True,
+    )
 
     broker.run()
diff --git a/test/unit/test_broker.py b/test/unit/test_broker.py
index 50d8449..bd6b002 100644
--- a/test/unit/test_broker.py
+++ b/test/unit/test_broker.py
@@ -1,8 +1,12 @@
 """M0.4 -- embedded broker: random ports, loopback default, clean stop."""
 
-from digitaltwin import ZMQ_BrokerProcess
+import asyncio
+
+from digitaltwin import DataType, ZMQ_BrokerProcess, connect_stream_client
 from digitaltwin.config import DEFAULT_BIND_HOST
 
+PING = DataType("ping")
+
 
 async def test_start_reports_bound_random_loopback_ports():
     proc = ZMQ_BrokerProcess()
@@ -23,15 +27,18 @@ async def test_start_reports_bound_random_loopback_ports():
 
 async def test_start_stop_cycling():
     proc = ZMQ_BrokerProcess()
-    seen = []
 
     for _ in range(3):
-        addrs = await proc.start()
+        pub_addr, sub_addr = await proc.start()
         assert proc.is_alive()
-        seen.append(addrs)
+        assert int(pub_addr.rsplit(":", 1)[1]) > 0
+        assert int(sub_addr.rsplit(":", 1)[1]) > 0
 
-        # start is idempotent while running
-        assert await proc.start() == addrs
+        # start is idempotent while running, and concurrent starts must not
+        # spawn a second broker
+        assert set(await asyncio.gather(proc.start(), proc.start())) == {
+            (pub_addr, sub_addr)
+        }
 
         await proc.stop()
         assert not proc.is_alive()
@@ -40,8 +47,21 @@ async def test_start_stop_cycling():
         # stop is idempotent
         await proc.stop()
 
-    # random ports: a fresh cycle does not reuse the previous bind
-    assert len(set(seen)) == len(seen)
+    # and a broker from a fresh cycle actually proxies
+    addrs = await proc.start()
+    try:
+        client = await connect_stream_client("cycled", *addrs)
+        queue: asyncio.Queue = asyncio.Queue()
+
+        await client.subscribe_to_dtype(PING, queue)
+        await asyncio.sleep(0.2)
+        await client.publish(PING, "alive")
+
+        assert (await asyncio.wait_for(queue.get(), 5.0)).data == "alive"
+        await client.close()
+
+    finally:
+        await proc.stop()
 
 
 async def test_configured_addresses_are_honoured():
diff --git a/test/unit/test_runtime_stop.py b/test/unit/test_runtime_stop.py
index 36ef7fd..dff7b08 100644
--- a/test/unit/test_runtime_stop.py
+++ b/test/unit/test_runtime_stop.py
@@ -116,6 +116,53 @@ async def test_stop_abandons_tasks_that_ignore_cancellation(
     await asyncio.sleep(0.05)
 
 
+async def test_concurrent_stop_joins_one_teardown(stream_clients, no_task_leaks):
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    runtime.start()
+    await asyncio.sleep(0.1)
+
+    await asyncio.gather(runtime.stop(), runtime.stop(), runtime.stop())
+
+    # every caller returned only after the one teardown finished
+    assert runtime.state is RuntimeState.STOPPED
+    assert runtime.running_tasks == set()
+    assert client._backend._ctx.closed
+
+
+async def test_graph_cannot_be_changed_after_stop(stream_clients):
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.start()
+    await runtime.stop()
+
+    with pytest.raises(RuntimeError):
+        runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    with pytest.raises(RuntimeError):
+        runtime.add_investigator(Forever(NO_FLOW), TICK, NULL_DTYPE)
+    with pytest.raises(RuntimeError):
+        runtime.add_agent(Forever(NO_FLOW), TICK, NULL_DTYPE)
+    with pytest.raises(RuntimeError):
+        runtime.add_barrier(Barrier("b"))
+
+
+async def test_stream_failure_fails_the_twin(stream_clients):
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+    runtime.start()
+
+    # what a dead receive loop reports
+    client._backend._report_error(RuntimeError("stream receive loop exited"))
+
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "RuntimeError: stream receive loop exited"
+
+    with pytest.raises(RuntimeError):
+        runtime.start()
+
+    await runtime.stop()
+
+
 async def test_component_failure_sets_failed_state(stream_clients):
     runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
     runtime.add_task(Boom(NO_FLOW), TRUTHY, NULL_DTYPE)
diff --git a/test/unit/test_streaming.py b/test/unit/test_streaming.py
index 75b362d..12774ae 100644
--- a/test/unit/test_streaming.py
+++ b/test/unit/test_streaming.py
@@ -72,6 +72,63 @@ async def test_unsubscribe_dtype_stops_delivery(stream_clients):
     await twin.subscribe_to_dtype(SENSOR, queue)
 
 
+def test_namespace_must_not_alias_topics():
+    for namespace in ("", "twin/a", "twin\x00a"):
+        with pytest.raises(ValueError):
+            PubSubClient(backend=None, namespace=namespace)
+
+
+async def test_receive_loop_survives_bad_payloads_and_callbacks(stream_clients):
+    """A malformed message or a raising callback must not stall the stream."""
+
+    twin = await stream_clients("twin-a")
+    backend = twin._backend
+    queue: asyncio.Queue = asyncio.Queue()
+
+    async def raises(message):
+        raise ValueError("bad callback")
+
+    await backend.subscribe(twin.topic(SENSOR), raises)
+    await twin.subscribe_to_dtype(SENSOR, queue)
+    await asyncio.sleep(0.2)
+
+    # not a pickle
+    await backend.pub_soc.send_multipart([twin.topic(SENSOR).encode(), b"garbage"])
+    await asyncio.sleep(0.2)
+
+    # the raising callback fires on this one, the queue still gets it
+    await twin.publish(SENSOR, "good")
+    assert (await _drain(queue)).data == "good"
+
+    assert not backend._task.done()
+
+
+async def test_unexpected_receive_loop_exit_is_reported(stream_clients):
+    twin = await stream_clients("twin-a")
+    backend = twin._backend
+
+    seen: list[BaseException] = []
+
+    def report(exc):
+        seen.append(exc)
+
+    twin.on_error = report
+    assert backend.on_error is report
+
+    # a receive loop which ends without close() means a silent stall
+    ended = asyncio.create_task(asyncio.sleep(0))
+    await ended
+    backend._run_done(ended)
+
+    assert len(seen) == 1
+    assert isinstance(seen[0], RuntimeError)
+
+    # ... but the loop ending as part of close() is not an error
+    await twin.close()
+    backend._run_done(ended)
+    assert len(seen) == 1
+
+
 async def test_close_releases_task_sockets_and_context(broker, no_task_leaks):
     from digitaltwin import connect_stream_client
 

From d8e42f4fa76a81b9172ca04fa8fdea01337dcd09 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sun, 16 Aug 2026 11:07:00 +0200
Subject: [PATCH 08/19] docs: add the DTaaS v1 plan and the P0
 liveness-scoped-calls plan

The reviewed plan documents the milestone series (M0-M3) references;
PR descriptions point at docs/dtaas-v1-plan.md as the authoritative
spec.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/dtaas-v1-plan.md                       | 557 ++++++++++++++++++++
 docs/orbit-p0-liveness-scoped-calls-plan.md | 147 ++++++
 2 files changed, 704 insertions(+)
 create mode 100644 docs/dtaas-v1-plan.md
 create mode 100644 docs/orbit-p0-liveness-scoped-calls-plan.md

diff --git a/docs/dtaas-v1-plan.md b/docs/dtaas-v1-plan.md
new file mode 100644
index 0000000..42afe34
--- /dev/null
+++ b/docs/dtaas-v1-plan.md
@@ -0,0 +1,557 @@
+# DT-as-a-Service (DTaaS) — v1 implementation plan
+
+Status: five independent Fable review rounds (2026-08-14/15), all
+findings folded in; compute-placement decision benchmarked; API model
+settled: short synchronous verbs + async `twin_create` (rhapsody
+pattern). Target repo:
+`radical/digital_twins` (package `digitaltwin`). Related plan:
+`orbit-p0-liveness-scoped-calls-plan.md` (P0) — **decoupled**: DTaaS
+v1 does not depend on it; interim limitations while it is unmerged
+are marked [P0-interim] below.
+
+## 1. Goal and target semantics
+
+Expose the experimental Digital Twin framework (this repo) as a
+long-running service: an ORBIT plugin hosted on a standalone,
+persistent ORBIT broker (driving use cases: AmSC / Matey fine-tuning,
+xGFabric). In production the broker runs on capable dedicated
+hardware. Endpoint-hosted deployment must remain possible
+([P0-interim]: until P0 lands that mode sits under ORBIT's 30 s
+relay backstop — harmless for the short verbs, limiting only
+`get_inference`).
+
+Agreed semantics (decisions, not open questions):
+
+- **DTaaS is long-running; twins come and go.** Twins are defined and
+  managed *programmatically* by clients (no declarative twin spec in
+  v1). A serializable graph/twin description is welcome as
+  introspection but must not constrain semantics.
+- **n twin instances per plugin session.** A session belongs to one
+  client; it hosts many independent twins. Twin teardown must not
+  disturb sibling twins or the session.
+- **Twins survive disappearing clients, without timeout.** Twins may
+  run for days; clients attach/detach opportunistically. Sessions are
+  therefore forced `persistent` server-side; reattach uses the sid as
+  a bearer capability (ownership check relaxed within the
+  single-token trust domain); orphans are discoverable via the admin
+  listing and killable via the ordinary teardown routes. Explicit
+  lifecycle only — no idle expiry.
+- **Recovery of twins across broker restart is out of scope for v1**
+  (candidate for v2/v3). Design state to be serializable where cheap.
+- **Streams (pubsub)**: two stages. Stage 1 (v1): the DT framework's
+  own ZMQ pubsub broker, run by the plugin, separate from ORBIT
+  messaging. Stage 2: an ORBIT-pubsub backend behind the same
+  abstraction — promoted to stretch milestone M3 for security
+  reasons (risk R7), required before production. The pubsub
+  abstraction is a deployment-time backend choice at the same
+  architectural altitude as RHAPSODY (compute backends); the
+  backend *interface* is the seam — nothing above it may depend on
+  ZMQ specifics.
+- **All user compute goes through the Rhapsody abstraction** (decided
+  after benchmarking, §6): asyncflow engines are constructed with
+  rhapsody's `OrbitExecutionBackend` targeting registered endpoints;
+  a co-located endpoint (same node as the broker) is the "local"
+  deployment. The broker process runs only the DT control plane. No
+  in-process `ProcessPoolExecutor` for user tasks.
+- **Persistent DT tasks (sensors, in-situ loops) run in the plugin
+  host process in v1** — as plain async `main_loop` code on the host
+  loop, using an injected, namespaced stream client (M0.3).
+  Persistent bodies are NOT `@flow.function_task`s: under an
+  Orbit-backed engine a function task would be cloudpickled to the
+  endpoint and occupy a backend slot for the twin's lifetime. This
+  simplifies the user API overall (today's hand-built task wrapping
+  and ZMQ clients disappear; `RuntimeAPI` gains its first publish
+  path). User code on the host loop (main_loops, callbacks,
+  selectors) is contractually thin async glue — documented (risk
+  R2), with one cheap guard: warn at instantiation if a persistent
+  component registered `function_task`s (catches the actual
+  migration mistake). Remote persistent components (psij
+  child-endpoint story) are post-v1.
+- **Ex-situ learning uses ROSE in v1**, as a *plugin-local module*
+  (the ROSE "raas" service plugin is abandoned; do not depend on it).
+  ROSE's learner engine is `OrbitExecutionBackend`-backed like
+  everything else, typically targeting a remote HPC endpoint. ROSE
+  `StreamingActiveLearner` (PR #98, commit 64330d9) is an accepted
+  dependency.
+- **Trust model**: clients ship code (cloudpickle) that executes in
+  the service. Accepted inside ORBIT's single-token trust domain
+  (same stance as rhapsody function tasks). Per-tenant auth is
+  post-v1. The DT *data plane* is currently weaker than that domain —
+  see risk R7 and milestone M3.
+
+## 2. Codebase facts the plan builds on
+
+Verified 2026-08-14/15 against: `digital_twins` @ `main` (df3b664),
+`radical.orbit` @ `devel` (8f1d18c), `rose` @
+`feature/streaming_learner` (64330d9 — branch, not merged),
+`rhapsody` @ dev/feature branches containing
+`backends/execution/orbit.py`. Independently re-verified by four
+fresh-eyes review agents.
+
+DT framework (`src/digitaltwin`, ~1200 lines):
+
+- `DTRuntime(flow: WorkflowEngine, streamer: PubSubClient)`
+  (`runtime.py:151`) never touches ZMQ directly; the only streaming
+  call is `streamer.subscribe_to_dtype` (`runtime.py:370`).
+- **No `stop()`**: `start()` only sets an event; `running_tasks` are
+  never cancelled; demos tear down via `flow.shutdown()` only. The
+  existing done-callbacks call `.result()` before discarding
+  (`runtime.py:193-197`, `runtime.py:68-72`, `components.py:209`) and
+  are not cancellation-safe; component exceptions surface there as
+  loop log noise, invisible to any state machine.
+- **Topics are un-namespaced**: `"runtime/dtypes/<label>"`
+  (`streaming.py:178`). Two twins using the same dtype label share a
+  topic string and cross-subscribe — the multi-tenancy blocker,
+  fixed by namespacing. (ZMQ SUBSCRIBE prefix-matching,
+  `streaming.py:142`, wastes bandwidth on label-prefix overlap, but
+  delivery is filtered by an exact dict lookup at `streaming.py:156`
+  — the topic terminator in M0.2 is hygiene, not correctness.)
+- **Transport leaks into user tasks**: every persistent task hand-
+  builds `ZMQ_PS_Client("tcp://127.0.0.1:5000")` inside its
+  `@flow.function_task` body (e.g.
+  `test/01-start-inference-stop/sensor.py:22-32`); addresses are
+  hardcoded literals repo-wide; ports fixed 5000/5001/5555.
+- **Pubsub payloads are cloudpickle** on both publish and receive
+  (`streaming.py:132`, `:155`) — anyone who can reach the XSUB port
+  achieves code execution in every subscriber, token-free (risk R7).
+- `ZMQ_Broker.run()` (`streaming.py:60-69`) is a blocking
+  `zmq.proxy()` with no stop path and no random-port bind; as a
+  spawn-context subprocess its stop path is terminate/join (M0.4).
+- `PubSubClient.subscribe_to_dtype` silently no-ops on duplicate
+  dtype (`streaming.py:196`) — per-twin clients required.
+- **No stream-client teardown exists**: `ZMQ_PS_Client` has no
+  `close()` (forever `_run` task `streaming.py:119`, monitor sockets
+  never closed `:103-108`, per-client `zmq.asyncio.Context` never
+  terminated `:84`); `PubSubClient` has no per-dtype unsubscribe.
+  With per-twin clients and churning twins this leaks unboundedly —
+  fixed in M0.1.
+- Further latent bugs to fix in passing: un-awaited coroutine at
+  `runtime.py:296`; `WindowDataType.__eq__` compares a field to
+  itself (`components.py:55-60`); dead `_to_block` machinery
+  (`runtime.py:177-181`).
+- Existing single-session service prototype:
+  `src/digitaltwin/remote/{remote_service,client}.py` (ZMQ REQ/REP;
+  `package()` cloudpickles a component *class* + ctor args; service
+  instantiates injecting its own `flow`, `remote_service.py:79`).
+  Its README declares the ORBIT port as the intended future.
+- Components take one `flow` in their constructor and register
+  `@flow.function_task`s at construction — components are
+  inseparable from a live engine (this binds engine lifetime to
+  component lifetime: engines cannot be swapped under running twins,
+  see R8).
+- `pyproject.toml` declares zero dependencies (asyncflow, rhapsody,
+  pyzmq, cloudpickle are actually required); `__init__.py` empty; no
+  pytest suite (only runnable demos under `test/`).
+
+ORBIT (`radical.orbit`, star topology, msgpack envelopes, 4 MiB frame
+cap — the cap applies to *our control traffic* too, see M1):
+
+- Plugin model: subclass `Plugin` with `plugin_name`,
+  `session_class`, `client_class`; auto-registered at import;
+  out-of-tree discovery via entry-point group `radical.orbit.plugins`
+  (`plugin_host_base.py:114-129`). Broker loads with
+  `--plugins default,<name>`. `is_enabled(app)` gates hosting role.
+  `Plugin` already ships `unregister_session/{sid}` (no owner check,
+  `plugin_base.py:600-609`) and `list_sessions` — reused, not
+  duplicated, by our admin story.
+- **The broker plugin host runs on a dedicated non-main thread with
+  its own event loop** (`broker.py:477-488`); endpoint-hosted plugins
+  likewise run on the runtime's work-loop thread. Consequence:
+  radical.asyncflow 0.5.0 `WorkflowEngine.create()` **crashes there**
+  (`_setup_signal_handlers` catches only `NotImplementedError`, but
+  non-main threads raise `RuntimeError` from `add_signal_handler`;
+  reproduced). Upstream fix is prerequisite P1. Each hosted-plugin
+  call is dispatched as its own task (`broker.py:820`), so a slow
+  handler does not head-of-line-block other calls.
+- Sessions: per-plugin-instance state (`sid → PluginSession`), owner
+  = broker-verified `x-orbit-src`; lifetime policy is chosen by the
+  *client* at `register_session` (`plugin_base.py:389-429`) — a
+  plugin enforces its own policy by overriding
+  `_normalize_session_policy`; the owner check is overridable
+  (`_check_owner`, `plugin_base.py:559`). `persistent` sessions never
+  expire and are not reclaimed on owner loss. `PluginSession.close()`
+  is the teardown hook; plugin-wide `shutdown()` must cancel
+  background tasks.
+- Call timeouts: broker-hosted (`dst='broker'`) dispatch never enters
+  the broker correlation table — no broker-side deadline; the
+  consumer default is 600 s, per-call overridable.
+  Participant→participant relay carries a 30 s broker backstop
+  removed by P0. [P0-interim]: all DTaaS verbs are short except
+  `get_inference`, which defaults to a large finite wait and is the
+  only endpoint-hosted casualty of the backstop.
+- Eventing: `session.notify(topic, data)` → at-most-once,
+  per-subscriber bounded drop-oldest queues, broker-assigned `seq`
+  (gaps observable), no persistence (`replay` plugin opt-in).
+- ORBIT's rhapsody plugin batches completion notifications with a
+  hardcoded `NOTIFY_BATCH_WINDOW = 0.25`, flushed server-side
+  (`plugin_rhapsody.py:43`, `:343`) — ~250 ms on every sequential
+  task round-trip (§6). Prerequisite P2 makes it an endpoint-level
+  config knob.
+- TaskDispatcher is run-to-completion only; persistent remote
+  processes are psij child endpoints. Not used in v1.
+- No session persistence across broker restart; durable state is a
+  plugin's own job (task_dispatcher's atomic-JSON `_replay_state`
+  pattern is the reference).
+
+ROSE / rhapsody:
+
+- `StreamingActiveLearner(asyncflow, batch_size, max_wait, conflate,
+  sources)`: async `feed()`, sync `attach_source()`; window =
+  batch_size or
+  max_wait-since-first-item; conflation bounds the queue latest-wins;
+  no simulation task — stream windows are injected into training;
+  criterion is a publish gate (`on_model_ready`), not a terminator;
+  loop-free construction, but construct and run in the same loop;
+  requires a real `WorkflowEngine` (typeguard); async-only. Exists
+  only on branch `feature/streaming_learner` (64330d9) until PR #98
+  merges.
+- `test/rose_streaming/run_me.py` demonstrates the intended bridge:
+  learner inside a `ModelInvestigator`, fed from `ON_INPUT`,
+  `on_model_ready → runtime.publish_new_model`; a bootstrap model
+  must be published or inference deadlocks on `has_published_model`.
+  Caveat for M2: its learner tasks are shell commands with *local*
+  paths (`run_me.py:44-57`) — they cannot run on a remote endpoint
+  unmodified (see M2 item 11).
+- rhapsody `OrbitExecutionBackend`
+  (`rhapsody/backends/execution/orbit.py`): AsyncFlow-compatible
+  backend over its own `EndpointRuntime`; submission batching
+  (`batch_window`, hardcoded to 0 in our wiring), event-based waits,
+  endpoint auto-selection, cloudpickled function tasks with a
+  Python-version guard. Awaiting it can take up to `start_timeout +
+  init_timeout` (30 + 120 s defaults) — this runs inside
+  `twin_create`'s background initialization, never inside a held
+  request. It tears down on failed init only; nothing reconnects a
+  live backend if its endpoint restarts (risk R8).
+
+## 3. Architecture
+
+One ORBIT plugin **`dt`**, implemented in this repo as
+`src/digitaltwin/service/`, registered via the
+`radical.orbit.plugins` entry point. Upstream changes in v1 are
+limited to prerequisites P1-P2.
+
+- `PluginDT(Plugin)`: `plugin_name = 'dt'`, `session_class =
+  DTSession`, `client_class = DTClient`, `is_enabled` unconditional
+  (broker-hosted by default via `--plugins default,dt`;
+  endpoint-hosted possible). Overrides `_normalize_session_policy` to
+  force `lifetime='persistent'` and `_check_owner` so the sid acts as
+  a bearer capability (reattach by sid). Owns the embedded DT stream
+  broker: one `ZMQ_Broker` subprocess (spawn context, random or
+  configured ports reported back, terminate/join stop), started on
+  first need, shared by all sessions/twins, and **supervised**: a
+  trivial liveness check respawns it on the same reported ports if
+  it dies (clients auto-reconnect; silent stream stalls on days-long
+  twins are not acceptable). **Binding policy (R7):
+  loopback by default; non-loopback binds require explicit
+  configuration and documented firewalling — mandatory guidance for
+  any demo.**
+- `DTSession(PluginSession)`: holds `twin_id → TwinInstance` and
+  session-shared, `OrbitExecutionBackend`-backed engines, keyed by
+  name in config: M1 creates exactly one, `'task'` (twin components;
+  typically a co-located endpoint); M2 adds `'exsitu'` (ROSE
+  learners; typically a remote HPC endpoint) as a non-breaking
+  config addition. One engine per name per session, never per twin;
+  engines initialize in `twin_create`'s background phase. Engine
+  configuration: per engine name `{endpoint_name, backends}` plus
+  plugin-level broker URL/token resolution; co-located-demo default
+  `backends=['concurrent']` (the `dragon_v3` default would break
+  demos); `batch_window` hardcoded 0. `close()` stops all twins,
+  then shuts engines down — bounded by `wait_for`-wrapping
+  `flow.shutdown()`, which is itself unbounded.
+- **Engine selection**: none in M1 — graph verbs take no engine
+  parameter; every component gets the `'task'` engine as its `flow`.
+  Dual-engine injection exists only for `StreamingLearnerInvestigator`
+  and is introduced with it in M2 (the class itself is the marker;
+  the service detects it by subclass check). A user-facing `engine=`
+  selector appears only if a non-learner component ever needs the
+  remote engine. All examples/demos state their engine wiring
+  explicitly.
+- **API model (settled 2026-08-15, modeled on the rhapsody plugin —
+  a real precedent: its `register_session` route returns the sid
+  immediately and initializes in background while the *client
+  helper* waits with poll fallback).** Every wire call is short:
+  - `twin_create` is the one async verb: it registers the twin
+    (client-supplied uuid → retry-idempotent), kicks engine
+    initialization in a background task (up to ~150 s), and returns
+    immediately with state `initializing`. The client helper's
+    default is to poll `twin_list` until `ready`.
+  - All other verbs are plain short request/response returning the
+    resulting state. Graph verbs fast-fail with a clear "not ready"
+    error while the twin is `initializing` — they never wait.
+    No request IDs, no notifications, no verb batching (compatible
+    future additions if ever needed); `twin_list` polling is the
+    only observation mechanism in v1.
+  - **Idempotency under reconnect** (bearer-sid retry path): `start`
+    on a running twin, `stop` on a stopped twin, and `twin_close` on
+    a closed twin are safe no-ops returning current state; tested in
+    M1.
+  - `get_inference` takes an optional client-supplied timeout
+    ([P0-interim]: default large finite; post-P0: infinite) and is
+    the only potentially-long call.
+  - `stop` is **terminal** in v1: start-after-stop is a tested
+    error; restartable twins are documented future work.
+- `TwinInstance`: `DTRuntime` + its own `PubSubClient` (namespace =
+  twin id; **twin ids are client-supplied uuids, globally unique
+  across sessions** — the stream broker is shared plugin-wide, and
+  the uuid makes `twin_create` retry-idempotent) + config + state
+  machine `initializing → ready → running → stopped | failed`, with
+  a last-error field fed from the done-callback exception path.
+  `twin_close` on a running twin: best-effort cancel of in-flight
+  backend tasks (rhapsody cancel routes), then abandon with a
+  bounded timeout.
+- Routes (per-session): `twin_create/{sid}`, `twin_list/{sid}`,
+  `twin_close/{sid}/{twin_id}`, `twin_call/{sid}/{twin_id}` carrying
+  exactly one graph verb per request (`add_task`,
+  `add_investigator`, `add_agent`, `start`, `stop`, `describe`,
+  `get_inference`; `describe` returns the serializable graph dict —
+  human-readable rendering is a client concern). Barriers are
+  **local-API only in v1**: `add_barrier` is not a remote verb — the
+  `Barrier` object doesn't fit the `package()` class-shipping
+  convention (post-construction config whose return values the
+  client needs locally, live asyncio primitives), and no remote demo
+  uses it. One admin route: `admin/sessions` (all sessions + twins +
+  owner + age + state + last error); teardown reuses the ordinary
+  `twin_close`/`unregister_session` with a sid from the listing.
+  Artifacts: cloudpickle-base64 in JSON, `package()` class-shipping
+  carried over from `RemoteDTRuntime`, with a client-side size check
+  against the 4 MiB frame cap and the client's Python + cloudpickle
+  versions — the service rejects skew with a clear error.
+- Client: one class, `DTClient(PluginClient)` (sync), whose public
+  API *is* the `test/09-remote` shape: `get_plugin('dt') →
+  create_twin() → package/add_* → start()`.
+- Deployment assumption: ≥1 registered endpoint running the rhapsody
+  plugin (a co-located one suffices; benchmark used
+  `radical-orbit-endpoint -n <name> -p rhapsody`); Python versions
+  compatible for function tasks (backend guards this).
+
+## 4. Milestones
+
+### P — upstream prerequisites (small PRs, first work items)
+
+P1. **radical.asyncflow**: `WorkflowEngine` must be constructible on
+    a non-main-thread event loop — catch `RuntimeError` in
+    `_setup_signal_handlers` (or add `install_signal_handlers=False`);
+    unit test creating an engine on a secondary-thread loop; pin the
+    fixed version. Blocks M1 (both hosting modes).
+P2. **radical.orbit**: the rhapsody plugin reads its notify window
+    from endpoint config/env (default 0.25 unchanged; the DT `'task'`
+    endpoint sets 0). One small PR in one repo; endpoint-scoped is
+    the right granularity (the co-located `'task'` endpoint is
+    dedicated; the `'exsitu'` endpoint keeps the default — 250 ms is
+    noise under long training tasks). Blocks M1 acceptance, not M1
+    development.
+
+(P0 — liveness-scoped call correlation — is a separate, decoupled
+plan; see status header.)
+
+### M0 — framework hardening (this repo, no ORBIT)
+
+1. `DTRuntime.stop()`: cancel `running_tasks`, stop barrier loops,
+   unsubscribe topics; make
+   all done-callbacks cancellation-safe AND route their exceptions
+   into twin state (`failed` + last error) instead of loop log noise
+   (`runtime.py:193-197`, `runtime.py:68-72`, `components.py:209`).
+   Idempotent, per-twin, terminal (start-after-stop is a tested
+   error), never touches shared engines. Define stop semantics for
+   in-flight backend tasks: best-effort cancel, bounded abandon.
+   **Stream-client teardown**: add `ZMQ_PS_Client.close()` (cancel
+   `_run`, disable/close monitor sockets, term context) and
+   `PubSubClient.unsubscribe_dtype()`/`close()`, wired into
+   `stop()`/twin teardown. Ride-along fixes: un-awaited coroutine at
+   `runtime.py:296`, `WindowDataType.__eq__` self-comparison
+   (`components.py:55-60`), dead `_to_block` removal
+   (`runtime.py:177-181`).
+2. Topic namespacing: `PubSubClient(namespace=...)` →
+   `dt/<twin_id>/dtypes/<label>`. Correctness target: identical
+   dtype labels in two twins must not cross-subscribe (test exactly
+   that). Also add a topic terminator (bandwidth hygiene against ZMQ
+   prefix matching; not a correctness fix).
+3. **Persistent-component contract**: persistent `main_loop`s run
+   inline on the host loop and publish/subscribe through an
+   injected, namespaced, connected `PubSubClient` exposed on
+   `RuntimeAPI` — no `function_task` wrapping and no hand-built ZMQ
+   clients in persistent paths; stream addresses come from config,
+   not literals. (The `function_task` guard lives service-side in
+   M1: only there are instantiation and `is_persistent` visible in
+   one place.) Migrate all demos accordingly. (A *serializable* stream-endpoint descriptor
+   is deferred until the first out-of-process stream consumer
+   exists — v2 remote persistent components.)
+4. `ZMQ_Broker` embedding: spawn-context subprocess launcher,
+   random-port bind with bound ports reported back, loopback-default
+   binding (R7), terminate/join stop. Start/stop cycling test. (No
+   in-process steerable-proxy rework — the subprocess boundary is
+   the stop path.)
+5. Config + packaging: configurable addresses everywhere, real
+   dependency list in `pyproject.toml`, package exports,
+   `DTRuntime.describe() → dict` (serializable graph summary;
+   introspection only; supersedes `print_graph` as the wire format).
+6. Tests: pytest + tox scaffolding; unit tests for stop/teardown
+   (incl. failure→`failed` state, start-after-stop error, and a
+   **leak assertion**: no lingering tasks/sockets/contexts after
+   twin teardown), namespacing, injected stream client; migrated
+   demos.
+
+### M1 — the ORBIT plugin (needs P1; acceptance needs P2)
+
+7. `PluginDT` / `DTSession` / `DTClient` + routes as in §3:
+   forced-persistent sessions, sid-capability reattach,
+   `admin/sessions`, the API model of §3 (async `twin_create` with
+   background engine init + short verbs, one verb per `twin_call`,
+   idempotency semantics), `package()` size + version checks,
+   embedded DT broker management (loopback-default, supervised
+   respawn on same ports), the single `'task'` engine with the
+   name-keyed `{endpoint_name, backends}` config schema
+   (plugin-level broker URL/token; demo default
+   `backends=['concurrent']`; `batch_window=0` hardcoded), and the
+   service-side guard warning when a persistent component registered
+   `function_task`s.
+8. Port `test/09-remote` to the ORBIT client as the integration demo;
+   retire `src/digitaltwin/remote/` and `test/remote_service.py`
+   afterward.
+9. Integration test: broker + co-located rhapsody endpoint + plugin +
+   client; two concurrent twins in one session; independent teardown;
+   twin churn (create/close cycles) with leak assertion; `twin_close`
+   while an inference task is in flight; client disconnect → twins
+   keep running → reattach by sid; idempotent retry (`twin_create`
+   same uuid, `start`/`stop`/`twin_close` no-ops); component crash →
+   `failed` + last error via `twin_list`. Plus one endpoint-hosted
+   smoke test (create/list/close, no `get_inference`) so that mode
+   doesn't silently rot. Check the §6 benchmark harness into `perf/`
+   and keep it runnable against this test stack.
+
+### M2 — ROSE ex-situ (first task: confirm PR #98 merged, else pin 64330d9)
+
+10. `StreamingLearnerInvestigator` base class packaging the
+    `rose_streaming` wiring (fed from `ON_INPUT`; `on_model_ready →
+    publish_new_model`; bootstrap model handled; clean learner
+    shutdown on twin stop — introduce a component stop hook here if
+    task cancellation alone isn't clean, internal first). This
+    milestone adds the `'exsitu'` engine (non-breaking config
+    addition) and dual-engine injection: the service detects the
+    class by subclass check and instantiates it with the engine set
+    (learner tasks on `'exsitu'`, inference on `'task'`) — demo code
+    states this explicitly.
+11. Make learner tasks remote-executable: convert training/AL/
+    criterion tasks to cloudpickled function tasks (version-guarded)
+    or stage scripts via the ORBIT staging plugin — the
+    `rose_streaming` local-path shell tasks do not survive a remote
+    endpoint as-is. Integration test with distinct `'task'` and
+    `'exsitu'` endpoints. Service demo (MNIST or π).
+
+### M3 (stretch, security-motivated) — ORBIT-pubsub backend
+
+12. `PubSubBackend` adapter over ORBIT eventing behind the M0.3
+    seam: moves the DT data plane inside the token-authenticated WS
+    star, eliminating the open ZMQ ports (closes R7 properly).
+    Semantics fit (at-most-once, drop-oldest ≈ DT conflation); 4 MiB
+    frame cap; `replay` plugin for late joiners. Required before
+    production deployment; until then R7 mitigations apply.
+
+## 5. Deferred (v2/v3)
+
+- Moving the pubsub abstraction out of this repo to its shared
+  RHAPSODY-layer home (after M3 proves the second backend); a
+  serializable stream-endpoint descriptor arrives with the first
+  out-of-process stream consumer.
+- Per-session subprocess isolation of twin control planes (blast
+  radius; §7 R2).
+- **Scale-out: delegated DT hosting** — endpoint-hosted `dt` plugins
+  on compute nodes, with a thin broker-side placement/registry
+  front-end routing `twin_create` to a chosen endpoint and proxying
+  by twin id. v1 keeps the plugin transport-agnostic precisely to
+  enable this. Agreed direction (2026-08-14): stream ingress to
+  compute nodes via broker-proxied streams (and/or
+  outbound-subscribe backends, e.g. MQTT); twin-lifetime vs
+  allocation-lifetime mismatch is an accepted limitation handled by
+  endpoint deployment policies (persistent endpoints), not by twin
+  migration. Revisit when approaching the centralized broker's
+  performance ceiling.
+- Twin recovery across broker restarts (`describe()` +
+  atomic-JSON/`_replay_state` pattern makes it feasible); engine
+  rebuild/self-healing after endpoint loss belongs to the same
+  recovery umbrella (see R8).
+- Remote persistent components (psij child endpoints); TaskDispatcher
+  pool/pilot provisioning when endpoints don't pre-exist.
+- Declarative twin spec; per-tenant auth (blocked on ORBIT
+  per-participant identity); non-pickle data-plane wire format or
+  CURVE (alternative/complement to M3 for R7); general ORBIT
+  in-process participant transport for broker-hosted plugins (noted
+  upstream).
+- **Restartable twins** (`stop` is terminal in v1; re-arming TRUTHY
+  components needs design).
+- **Universal-async API mode** (verbs return request IDs, results
+  via notifications): considered and rejected for v1 (2026-08-15) —
+  complexity in the critical path for payoff that P0 provides
+  structurally and that synchronous verbs don't need. Adding fields
+  or an async mode later is a non-breaking JSON change. Verb
+  batching likewise deferred until a real latency need appears.
+
+## 6. Measured basis for the compute-placement decision
+
+Loopback benchmark 2026-08-14 (broker + rhapsody endpoint on one
+host, no-op asyncflow function task, 200 sequential calls). Exact
+configuration per row — **both orbit rows ran with client-side
+`batch_window=0`**; they differ only in the *server-side* notify
+window (`NOTIFY_BATCH_WINDOW`, patched on the endpoint for row 3):
+
+| path                                              | p50     | p99     | 50-concurrent  |
+|---------------------------------------------------|---------|---------|----------------|
+| in-process `ProcessPoolExecutor`                  | 10.9 ms | 14.1 ms |  35 tasks/s¹   |
+| orbit, `batch_window=0`, notify window 0.25 s     | 261 ms  | 269 ms  | 155 tasks/s    |
+| orbit, `batch_window=0`, notify window 0          | 20.2 ms | 23.2 ms | 459 tasks/s    |
+
+¹ Anomalously below the sequential rate (~92/s); likely pool/pickle
+contention under gather. Unexplained; re-measure before citing this
+cell elsewhere. The placement decision does not hinge on it.
+
+Full defaults (client batching 0.25 s *and* notify window 0.25 s)
+were not measured; both delayed flushes sit on the sequential path,
+so expect ~500 ms. The harness gets checked into `perf/` in M1
+(item 9).
+
+Conclusion: routing all compute through the Rhapsody abstraction
+costs ~9 ms p50 per sequential in-situ prediction and *wins* under
+concurrency; the 250 ms in row 2 is attributable to the server-side
+notify batcher alone (hence P2 remains an M1-acceptance gate).
+In-situ deadlines below ~20 ms would need in-process execution —
+no current use case is near that.
+
+## 7. Risks and dependencies
+
+- R1 **P1 is make-or-break**: without the asyncflow fix, engine
+  creation crashes in both hosting modes. First work item.
+- R2 **Shared-loop blast radius**: user glue code (main_loops,
+  callbacks) runs on the plugin host loop; a blocking tenant stalls
+  co-hosted plugins. v1: thin-glue contract documented + production
+  guidance (dedicated, adequately sized broker host — the planned
+  production brokers are). v2: subprocess isolation.
+- R3 **Immortal sessions leak by design**; mitigated by the
+  `admin/sessions` listing (discovery) + ordinary teardown routes,
+  and `twin_list` observability (incl. `failed` + last error).
+- R4 Cloudpickle class-shipping over ORBIT = arbitrary code execution
+  in the service, inside the single-token trust domain: documented,
+  accepted for v1.
+- R5 Version pins: ROSE `StreamingActiveLearner` = PR #98 / commit
+  64330d9 (branch until merged); rhapsody release containing
+  `OrbitExecutionBackend`; radical.orbit release containing P2;
+  radical.asyncflow release containing P1.
+- R6 AsyncFlow's own signal/shutdown paths must not race
+  `BrokerPluginHost.shutdown()`; engine teardown in
+  `DTSession.close()` is bounded by `wait_for`-wrapping
+  `flow.shutdown()` — asyncflow's own shutdown is an unbounded
+  gather (`workflow_manager.py:1777`), so a bare await would hang.
+- R7 **Token-free RCE on the DT data plane**: pubsub payloads are
+  cloudpickled; anyone reaching the XSUB port executes code in every
+  subscriber — weaker than the ORBIT trust domain. v1 mitigations
+  (mandatory, including for demos): loopback-default binding;
+  non-loopback binds require explicit config + firewalled/private
+  network, documented. Proper fix: M3 (ORBIT-pubsub backend) — and/or
+  non-pickle wire format/CURVE (§5) — required before production.
+- R8 **Endpoint loss strands engines**: `OrbitExecutionBackend` has
+  no reconnect, and components bind their engine at construction —
+  engines cannot be swapped under running twins. v1 remediation:
+  documented failure mode; the session is closed (admin listing +
+  `unregister_session`) and the client recreates its twins.
+  Self-healing belongs to the v2/v3 recovery umbrella.
+- Each milestone = one PR on a feature branch (`feature/dtaas-*`),
+  tests green before stacking the next.
diff --git a/docs/orbit-p0-liveness-scoped-calls-plan.md b/docs/orbit-p0-liveness-scoped-calls-plan.md
new file mode 100644
index 0000000..c98a075
--- /dev/null
+++ b/docs/orbit-p0-liveness-scoped-calls-plan.md
@@ -0,0 +1,147 @@
+# P0 — Liveness-scoped call correlation in ORBIT
+
+Status: reviewed (Fable subagent review 2026-08-15; findings folded
+in — the resulting change is *smaller* than the first sketch).
+Standalone `radical.orbit` PR against `devel`. **Ordering: decoupled
+from DTaaS** (2026-08-15): DTaaS v1 does not depend on P0 (see
+"Ordering and DTaaS consequences" below) — implement whenever
+convenient, before or after.
+
+## Problem
+
+A call timeout conflates two questions:
+
+1. *Is the other side still working?* (liveness) — the only thing an
+   intermediary legitimately cares about, since it must GC
+   correlation state for dead parties.
+2. *How long am I willing to wait?* (patience) — belongs exclusively
+   to the caller; no intermediary can answer it, so any fixed cap is
+   wrong for someone.
+
+ORBIT today uses wall-clock timers for both: relayed calls carry a
+hard broker backstop (`BrokerTuning.call_timeout = 30.0`,
+`broker.py:110`; deadline stamped for forwarded calls at
+`broker.py:838` **and** broker-originated calls at `broker.py:877`;
+enforced by the `_reap_calls` sweeper, `broker.py:1114-1131`), and
+the consumer side defaults to 600 s (`runtime.py:85`). Long-running
+plugin verbs are legitimate; every new one restarts the
+pick-a-bigger-number game.
+
+## Design
+
+Separate the two concerns. Review-refined; each element is the
+minimal mechanism found:
+
+- **Fail-inflight at the participant-removal chokepoint (the core
+  change).** `_fail_inflight_for` exists but fires only on the
+  `lost` transition (`broker.py:1061-1069`). Participants are also
+  removed by *clean close* (`_on_socket_drop` with `clean=True`,
+  `broker.py:1034-1038` — the common `endpoint.stop()` path) and by
+  operator `_disconnect` (`broker.py:1001-1022`); both currently
+  rely on the reaper. Move the `_fail_inflight_for` call into the
+  `_remove_participant` chokepoint (sync context — `spawn` it) so
+  all three removal paths fail in-flight calls uniformly. Then
+  delete the deadline stamps, the `_reap_calls` sweeper, the
+  `reap_interval` knob, and `_Call.deadline`. Caller-side cleanup on
+  clean disconnect must be **added** (today it exists only in the
+  lost path, `broker.py:1083-1085`).
+- **No new state.** No per-participant index: `_fail_inflight_for`'s
+  linear scan over `_calls` (capped at 4096, `broker.py:830-835`)
+  on a rare removal event is fine. No new error machinery:
+  `_fail_call` (`broker.py:1087-1101`) already synthesizes the right
+  response shape — adjust the reason (e.g. `participant-lost`).
+- **Suspect fast-fail.** Requests routed while the destination is
+  `suspect` land on a dead socket and are silently swallowed
+  (`_route_request` checks only registry presence `broker.py:823`;
+  `_send` swallows failures `broker.py:1210-1216`); if the endpoint
+  then *resumes* (resume keys keep `_calls` entries,
+  `broker.py:744-763`), the call can never complete. Fix:
+  `_route_request` fails fast with a retryable 503 when
+  `liveness == 'suspect'`; treat a synchronous send failure the same
+  way.
+- **Consumer-side mirror: resume-key-mismatch flush.** The runtime
+  keeps pending futures across reconnect (`runtime.py:524-528`) and
+  silently accepts a fresh resume key from a restarted broker
+  (`runtime.py:643`) whose correlation state is gone — permanent
+  hangs under infinite patience. Fix: on re-register, if the
+  presented resume key was not honored (fresh registration), fail
+  all `_pending` futures with a connection-reset-style error.
+- **Timeout knob split (client-owned patience ≠ served-handler
+  bound).** `_call_timeout` is dual-use: consumer patience
+  (`runtime.py:827`) *and* the served-handler bound
+  (`_serve_request` wraps dispatch in `wait_for`, 504s on expiry and
+  frees the admission slot, `runtime.py:735-755`). Split into two
+  knobs:
+  - consumer patience: **default infinity** (agreed 2026-08-15);
+    per-call override unchanged; make the timeout arithmetic
+    None-safe (`fut.result(timeout + 5.0)` at `runtime.py:827-833`
+    crashes on `None` today).
+  - served-handler bound: **stays finite** (600 s default,
+    configurable). This preserves the invariant that every admitted
+    request eventually gets *some* response — which is what keeps
+    the broker's call table self-cleaning without the reaper (a
+    black-hole endpoint would otherwise accumulate entries to the
+    4096 cap and 503 all relayed calling), and frees admission
+    slots under wedged handlers. Consistent with DTaaS's design:
+    handlers are short by construction; long work runs in
+    background tasks.
+- **Call-site audit.** In-tree callers relying on defaults:
+  task_dispatcher's `_CallerSyncHTTP` defaults `timeout=None`
+  (`plugin_task_dispatcher.py:102`, `:117`, `:686`) — today bounded
+  only by the 30 s backstop; post-change a stuck child would pin an
+  `asyncio.to_thread` worker, so give these explicit timeouts.
+  Gateway HTTP path keeps its own explicit 600 s
+  (`gateway.py:91,590-593`) — out of scope, unchanged.
+- **Residual gap, explicitly accepted:** a handler deadlocked inside
+  a live process is caught by the served-handler bound, not by
+  liveness. Per-request progress frames remain a possible later
+  refinement, not part of P0.
+
+Non-changes verified by review: `dst='broker'` dispatch never enters
+the correlation table (`broker.py:819-821`) — unchanged; corr_ids
+are uuid4 (`protocol.py:74-80`) — reuse is a non-issue.
+
+## Tests / acceptance
+
+New:
+- Relayed call whose handler sleeps > 30 s completes with no caller
+  timeout set (but under the served-handler bound).
+- Serving endpoint **killed** mid-call → prompt `participant-lost`
+  error (liveness window, not wall clock).
+- Serving endpoint **clean-closes** (`stop()`) mid-call → same.
+- Operator `disconnect` of the serving endpoint mid-call → same.
+- Call issued while dst is `suspect` → fast 503; call issued
+  *before* suspect completes after resume (regression: entries
+  survive resume).
+- Broker-originated call (`BrokerCaller` future) on endpoint death →
+  future resolves with the error; thread-safe caller unblocked.
+- Broker restart mid-call → consumer's pending futures fail with
+  connection-reset (resume-key-mismatch flush).
+- Caller-side timeout → broker call table does not grow
+  monotonically (self-cleaning via served-handler bound).
+- Caller disconnects mid-call → its entries dropped (both clean and
+  lost paths).
+
+Existing-suite touchpoints: `test_broker.py:667-693` (reaper test —
+repurpose as "entry survives past old deadline while dst alive"),
+`test_broker.py:71-72` (fixture tuning keys), `test_runtime.py:381-397`
+(served-handler bound keeps honoring its knob), `test_gateway.py:403-404`
+(unchanged). Re-run task_dispatcher and rhapsody-plugin integration
+tests (both rely on relayed calls).
+
+## Ordering and DTaaS consequences
+
+DTaaS v1 does **not** depend on P0: its target deployment is
+broker-hosted, and `dst='broker'` dispatch carries no broker-side
+deadline today; the 600 s consumer default (per-call overridable)
+covers DTaaS's slow verbs; the rhapsody backend already operates
+under the 30 s relay backstop (long operations are short calls +
+event waits — proven by the 2026-08-14 benchmark on vanilla ORBIT).
+
+Until P0 lands:
+- Endpoint-hosted DTaaS keeps the 30 s relay constraint (documented
+  secondary-mode limitation).
+- `get_inference` defaults to a large finite wait, not infinite.
+
+After P0: those two lines are deleted from the DTaaS plan; nothing
+else there changes.

From 2aba3e4a49302e250ebe6d09aadc2a71824b279b Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Mon, 17 Aug 2026 16:27:30 +0200
Subject: [PATCH 09/19] A stream endpoint a task can carry: PubSubConfig

Upstream review (Ben): handing components the live streamer is wrong for
anything that does not run in the host process -- the client owns
sockets, a receive loop and subscriber queues, none of which a remote
machine can reach.  This is the stream-endpoint descriptor the plan cut
in review round five, back in its minimal form.

- `PubSubConfig` is a frozen dataclass of plain fields (namespace,
  publish/subscribe address, backend kind) and therefore travels as
  pickle or as a dict.  `connect(timeout=None)` opens the backend for
  its kind, connects it bounded, and wraps it in a namespaced
  PubSubClient; a client which fails to connect is closed before the
  error propagates.  `resolve()` builds the config for the configured
  broker, and `connect_stream_client()` is now that plus connect.
- A backend declares which kind reopens it (`PubSubBackend.kind`, `zmq`
  here) and exposes the addresses a remote client would dial; a config
  for a foreign kind is refused rather than mis-opened.  The field is a
  plain string, so a second backend needs no change here.
- `PubSubClient.config` describes the client's own endpoint;
  `DTRuntime.stream_config` derives the twin's from the injected client
  (so it cannot go stale) and `RuntimeAPI` exposes both: `stream` stays
  exactly as it was -- the in-process convenience -- and `stream_config`
  is what a task ships when its code runs elsewhere.

Where shipping it off-host is documented, so is what it implies: the
broker has to be reachable from there, which for zmq means a
non-loopback bind on a firewalled network, not the loopback default.

API compatible: demos and tests are untouched except for new tests.
25 passing; demo 01 re-run live on non-default ports (env-resolved).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/__init__.py      |  2 +
 src/digitaltwin/runtime.py       | 27 +++++++++-
 src/digitaltwin/streaming.py     | 91 +++++++++++++++++++++++++++++---
 test/unit/test_runtime_stream.py | 27 ++++++++++
 test/unit/test_streaming.py      | 54 ++++++++++++++++++-
 5 files changed, 193 insertions(+), 8 deletions(-)

diff --git a/src/digitaltwin/__init__.py b/src/digitaltwin/__init__.py
index 3253440..e88a0a4 100644
--- a/src/digitaltwin/__init__.py
+++ b/src/digitaltwin/__init__.py
@@ -18,6 +18,7 @@
 from .streaming import (
     PubSubBackend,
     PubSubClient,
+    PubSubConfig,
     ZMQ_Broker,
     ZMQ_BrokerProcess,
     ZMQ_PS_Client,
@@ -33,6 +34,7 @@
     "ModelInvestigator",
     "PubSubBackend",
     "PubSubClient",
+    "PubSubConfig",
     "RuntimeAPI",
     "RuntimeState",
     "SciAgent",
diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index 60ab0ea..a0a828c 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -20,7 +20,7 @@
     UtilityTask,
     _TwinComponent,
 )
-from .streaming import PubSubClient
+from .streaming import PubSubClient, PubSubConfig
 
 logger = logging.getLogger(__name__)
 
@@ -83,10 +83,26 @@ def stream(self) -> PubSubClient:
         Persistent components publish their output through it (the runtime
         subscribes to that dtype and feeds the graph with it).  Components
         never build their own transport clients and never see addresses.
+
+        This is the in-process convenience: the same endpoint `stream_config`
+        describes, already open.  Code which does not run on the host loop
+        needs the config instead -- see below.
         """
 
         return self._runtime.streamer
 
+    @property
+    def stream_config(self) -> PubSubConfig:
+        """The twin's stream endpoint as plain data.
+
+        Ship *this* to code which runs outside the host process (a task in
+        another process or on another host) and let it open its own client:
+        the live client above holds sockets, a receive loop and subscriber
+        queues, none of which can travel.
+        """
+
+        return self._runtime.stream_config
+
     def subscribe_to_topic(self, topic: str, task: Callable):
         self._ant.subscriptions[topic].append(task)
 
@@ -210,6 +226,15 @@ def __init__(self, flow: WorkflowEngine, streamer: PubSubClient):
         # a stalled stream is a twin failure, not a log line
         streamer.on_error = self._record_error
 
+    @property
+    def stream_config(self) -> PubSubConfig:
+        """This twin's stream endpoint as plain data (see `PubSubConfig`).
+
+        Derived from the injected client, so it cannot go stale.
+        """
+
+        return self.streamer.config
+
     def start(self):
         if self.state is RuntimeState.STOPPED:
             raise RuntimeError("stop() is terminal - this twin cannot be restarted")
diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index 7ae5445..2eb8233 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -8,6 +8,7 @@
 import multiprocessing
 
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from typing import Callable, Optional
 
 import cloudpickle
@@ -29,6 +30,15 @@
 class PubSubBackend(ABC):
     label = "generic"
 
+    # names this backend in a PubSubConfig: what has to reopen the
+    # endpoint.  Every backend declares its own.
+    kind = "generic"
+
+    # the addresses another process would connect to.  Part of the backend
+    # interface because a PubSubConfig is built from them.
+    pub_addr: Optional[str] = None
+    sub_addr: Optional[str] = None
+
     def __init__(self):
         # asynchronous failures (a dead receive loop above all) are reported
         # here -- see PubSubClient.on_error.  A silently stalled stream is
@@ -225,6 +235,7 @@ def _stop(self, timeout):
 
 class ZMQ_PS_Client(PubSubBackend):
     label = "local"
+    kind = "zmq"
 
     def __init__(self, pub_addr: Optional[str] = None, sub_addr: Optional[str] = None):
         super().__init__()
@@ -430,6 +441,17 @@ def on_error(self) -> Optional[Callable[[BaseException], None]]:
     def on_error(self, callback: Optional[Callable[[BaseException], None]]):
         self._backend.on_error = callback
 
+    @property
+    def config(self) -> "PubSubConfig":
+        """This client's endpoint as plain, shippable data."""
+
+        return PubSubConfig(
+            namespace=self.namespace,
+            pub_addr=self._backend.pub_addr,
+            sub_addr=self._backend.sub_addr,
+            kind=self._backend.kind,
+        )
+
     def topic(self, dtype: DataType) -> str:
         return f"dt/{self.namespace}/dtypes/{dtype.name}{self.TOPIC_TERMINATOR}"
 
@@ -479,14 +501,71 @@ async def close(self):
         await self._backend.close()
 
 
+@dataclass(frozen=True)
+class PubSubConfig:
+    """A twin's stream endpoint as plain data: what to open, and where.
+
+    A live `PubSubClient` cannot travel -- it owns sockets, a receive loop
+    and subscriber queues, all of which are process-local.  Code which
+    runs outside the host process (a task in another process, or on
+    another host) therefore receives this description instead, ships it
+    along as pickle or JSON, and opens its own client with `connect()`.
+
+    Shipping it off-host implies the broker is reachable from there.  For
+    the `zmq` backend that means the broker was bound to a non-loopback
+    address on a private, firewalled network -- the default loopback bind
+    is deliberately not reachable from another host (see the binding
+    policy in the README).
+
+    `kind` names the backend which has to open the endpoint; a backend
+    declares its own (`PubSubBackend.kind`).  It stays a plain string so a
+    further backend needs no change here.
+    """
+
+    namespace: str
+    pub_addr: Optional[str] = None
+    sub_addr: Optional[str] = None
+    kind: str = ZMQ_PS_Client.kind
+
+    @classmethod
+    def resolve(
+        cls,
+        namespace: str,
+        pub_addr: Optional[str] = None,
+        sub_addr: Optional[str] = None,
+    ) -> "PubSubConfig":
+        """Describe the configured broker: explicit addresses, else the
+        environment, else the loopback defaults (see `config`)."""
+
+        return cls(namespace, *stream_addresses(pub_addr, sub_addr))
+
+    async def connect(self, timeout: Optional[float] = None) -> PubSubClient:
+        """Open a connected, namespaced client for this endpoint.
+
+        Bounded by `timeout` (None waits forever).  A client which fails to
+        connect is closed before the error propagates, so an unreachable
+        broker leaks neither sockets nor a context.
+        """
+
+        if self.kind != ZMQ_PS_Client.kind:
+            raise ValueError(
+                f"cannot open a {self.kind!r} stream endpoint here:"
+                f" no backend of that kind is available"
+            )
+
+        backend = ZMQ_PS_Client(self.pub_addr, self.sub_addr)
+        try:
+            await asyncio.wait_for(backend.connect(), timeout)
+        except BaseException:
+            await backend.close()
+            raise
+
+        return PubSubClient(backend, self.namespace)
+
+
 async def connect_stream_client(
     namespace: str, pub_addr: Optional[str] = None, sub_addr: Optional[str] = None
 ) -> PubSubClient:
     """Build and connect a namespaced stream client from configuration."""
 
-    pub_addr, sub_addr = stream_addresses(pub_addr, sub_addr)
-
-    backend = ZMQ_PS_Client(pub_addr, sub_addr)
-    await backend.connect()
-
-    return PubSubClient(backend, namespace)
+    return await PubSubConfig.resolve(namespace, pub_addr, sub_addr).connect()
diff --git a/test/unit/test_runtime_stream.py b/test/unit/test_runtime_stream.py
index 8e09e5a..ebdcfcb 100644
--- a/test/unit/test_runtime_stream.py
+++ b/test/unit/test_runtime_stream.py
@@ -71,6 +71,33 @@ async def wait_for_data():
     assert all(value % 2 == 0 for value in received)
 
 
+class ConfigProbe(UtilityTask):
+    def __init__(self, flow, seen):
+        super().__init__(flow)
+        self.seen = seen
+
+    async def main_loop(self, runtime, in_data):
+        self.seen.append((runtime.stream_config, runtime.stream))
+
+
+async def test_components_see_the_stream_config(stream_clients):
+    """What a component gets handed: the live client for in-process use,
+    and the same endpoint as plain data for anything that has to travel."""
+
+    seen: list = []
+    client = await stream_clients("twin-a")
+
+    runtime = DTRuntime(None, client)
+    runtime.add_task(ConfigProbe(None, seen), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    await asyncio.sleep(0.1)
+    assert seen == [(client.config, client)]
+    assert runtime.stream_config == client.config
+
+    await runtime.stop()
+
+
 async def test_two_twins_on_one_broker_stay_separate(stream_clients, no_task_leaks):
     """Same component code, same dtype labels, two namespaces: the twins
     must not see each other's data."""
diff --git a/test/unit/test_streaming.py b/test/unit/test_streaming.py
index 12774ae..5bb153d 100644
--- a/test/unit/test_streaming.py
+++ b/test/unit/test_streaming.py
@@ -1,10 +1,12 @@
 """M0.2 -- topic namespacing and stream client teardown."""
 
 import asyncio
+import dataclasses
+import pickle
 
 import pytest
 
-from digitaltwin import DataType, PubSubClient
+from digitaltwin import DataType, PubSubClient, PubSubConfig
 
 SENSOR = DataType("sensor")
 
@@ -129,6 +131,56 @@ def report(exc):
     assert len(seen) == 1
 
 
+async def test_stream_config_travels_and_reopens_the_endpoint(broker, stream_clients):
+    """A task which cannot hold the live client gets the config instead."""
+
+    twin = await stream_clients("twin-a")
+    config = twin.config
+
+    assert config.namespace == "twin-a"
+    assert config.kind == "zmq"
+    assert (config.pub_addr, config.sub_addr) == broker.get_connection_str()
+
+    # plain data: it travels as pickle (cloudpickled task bodies) and as a
+    # dict (JSON on a wire)
+    assert pickle.loads(pickle.dumps(config)) == config
+    assert dataclasses.asdict(config) == {
+        "namespace": "twin-a",
+        "pub_addr": config.pub_addr,
+        "sub_addr": config.sub_addr,
+        "kind": "zmq",
+    }
+
+    # ... and the far end opens its own client from it, in the same
+    # namespace, reaching the twin
+    revived = pickle.loads(pickle.dumps(config))
+    other = await revived.connect(timeout=10.0)
+    try:
+        queue: asyncio.Queue = asyncio.Queue()
+        await twin.subscribe_to_dtype(SENSOR, queue)
+        await asyncio.sleep(0.2)
+
+        await other.publish(SENSOR, "from a config")
+        assert (await _drain(queue)).data == "from a config"
+
+    finally:
+        await other.close()
+
+
+async def test_stream_config_rejects_a_foreign_backend_kind():
+    # the seam: another backend's endpoint is not this backend's to open
+    with pytest.raises(ValueError):
+        await PubSubConfig("twin-a", kind="orbit").connect()
+
+
+async def test_stream_config_connect_is_bounded_and_leaves_nothing(no_task_leaks):
+    # nothing listening on this port -- a connect must not park forever
+    config = PubSubConfig("twin-a", "tcp://127.0.0.1:1", "tcp://127.0.0.1:1")
+
+    with pytest.raises((asyncio.TimeoutError, TimeoutError)):
+        await config.connect(timeout=0.5)
+
+
 async def test_close_releases_task_sockets_and_context(broker, no_task_leaks):
     from digitaltwin import connect_stream_client
 

From 1fe23d5ace35ef5ef94974d3c4f2f0c9d1a43083 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Mon, 17 Aug 2026 18:00:39 +0200
Subject: [PATCH 10/19] A component failure stops the twin

`failed` meant "something broke and I recorded it": the twin's other
tasks kept running, its stream client kept its subscriptions, and the
whole lot stayed up until a client got around to closing the twin.  A
component failure is a twin failure -- the others have lost the graph
they were part of -- so it now runs the same teardown `stop()` runs and
ends in `failed` with `last_error`, not in `stopped`.

The traps this had to avoid:

- teardown cannot be awaited where the failure is seen.  `_record_error`
  is reached from synchronous done-callbacks (a component task, the
  stream backend's receive loop via `on_error`) and from inside the
  teardown itself, so it *schedules* one and keeps the handle.
- the teardown task must not be registered in `running_tasks`: teardown
  cancels that set, so it would cancel itself on its first await.
  `_start_teardown` creates it outside the set and consumes its
  exception, since nothing joins the teardown a failure started.
- one teardown only.  `_stop_task` is the single handle for both doors,
  so `stop()` on a failed twin joins the teardown already in flight
  (bounded by the same `STOP_TIMEOUT` the service would have granted)
  instead of starting a second one, and the state it finds is the state
  it leaves: `failed` sticks, which is the mirror image of the rule that
  a stopped twin stays stopped.
- the first error is the reported one.  Everything after it is fallout --
  teardown cancelling the failure's siblings, a stop hook tripping over a
  half-dead component -- and must not clobber the cause.
- `_to_asyncio_task` now refuses on a twin whose teardown is scheduled,
  not just on a stopped one: a task registered after teardown swapped
  `running_tasks` out would never be cancelled by anyone.

`test_component_failure_sets_failed_state` asserted the old contract
(`stopped` after an explicit stop of a failed twin) and is replaced by
`test_component_failure_tears_the_twin_down`, which makes the same leak
assertions the explicit-stop tests make.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/runtime.py     |  93 ++++++++++-
 test/unit/test_runtime_stop.py | 279 +++++++++++++++++++++++++++++++--
 2 files changed, 354 insertions(+), 18 deletions(-)

diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index a0a828c..93abdf8 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -29,7 +29,12 @@
 
 
 class RuntimeState(StrEnum):
-    """Lifecycle of a twin runtime.  `stopped` is terminal."""
+    """Lifecycle of a twin runtime.
+
+    `stopped` and `failed` are both terminal: a twin which fails tears
+    itself down (see `_record_error`), it just reports the error rather
+    than a clean stop.
+    """
 
     READY = "ready"
     RUNNING = "running"
@@ -221,6 +226,8 @@ def __init__(self, flow: WorkflowEngine, streamer: PubSubClient):
         self.state = RuntimeState.READY
         self.last_error: Optional[str] = None
 
+        # the one teardown, whichever door started it: stop() or a failure.
+        # Its presence is also what closes the twin for new work.
         self._stop_task: Optional[asyncio.Task] = None
 
         # a stalled stream is a twin failure, not a log line
@@ -262,17 +269,53 @@ async def stop(self, timeout: float = STOP_TIMEOUT):
         propagates the cancellation into the backend call, which is a
         best-effort cancel.  Whatever has not settled after `timeout` is
         abandoned with a warning -- stop() never waits unboundedly.
+
+        On a twin which already failed this is a bounded no-op: the
+        failure tore the twin down on its own, so stop() joins *that*
+        teardown (on its budget, not this `timeout`) and leaves the twin
+        `failed` -- the error is the more useful fact to report, and
+        `last_error` survives.
         """
 
         if self._stop_task is None:
             # flip the state before scheduling: no new work from here on
             self.state = RuntimeState.STOPPED
-            self._stop_task = asyncio.ensure_future(self._teardown(timeout))
+            self._stop_task = self._start_teardown(timeout)
 
         # every caller joins the one teardown; a cancelled caller does not
         # abort it (stop is terminal)
         await asyncio.shield(self._stop_task)
 
+    def _start_teardown(self, timeout: float) -> asyncio.Task:
+        """Schedule the one teardown and return its handle.
+
+        Deliberately *not* through `_to_asyncio_task`: teardown cancels
+        everything in `running_tasks`, so a teardown registered there
+        would cancel itself on its first await.
+
+        The handle is what makes teardown joinable (`stop()`) and
+        observable; the done-callback is what keeps it quiet when nobody
+        joins it, which is the case whenever a failure started it.
+        """
+
+        task = asyncio.ensure_future(self._teardown(timeout))
+        task.add_done_callback(self._teardown_done)
+
+        return task
+
+    def _teardown_done(self, task: asyncio.Task):
+        """Consume the teardown's outcome: nothing awaits the teardown a
+        failure started, and an unretrieved exception would surface as
+        loop noise long after the fact.  It is logged, not recorded -- a
+        mishap while cleaning up must not overwrite the cause."""
+
+        if task.cancelled():
+            return
+
+        exc = task.exception()
+        if exc is not None:
+            logger.error("twin teardown failed: %s", exc, exc_info=exc)
+
     async def _teardown(self, timeout: float):
         tasks, self.running_tasks = self.running_tasks, set()
         for task in tasks:
@@ -298,8 +341,11 @@ async def _call_await(self, func, *args, **kwargs):
         await func(*args, **kwargs)
 
     def _to_asyncio_task(self, func, *args, **kwargs) -> Optional[asyncio.Task]:
-        if self.state is RuntimeState.STOPPED:
-            logger.debug("twin is stopped - not running %s", func)
+        # a twin which is being torn down starts no new work: a task
+        # registered after teardown swapped `running_tasks` out would never
+        # be cancelled by anyone
+        if self._stop_task is not None or self.state is RuntimeState.FAILED:
+            logger.debug("twin is %s - not running %s", self.state, func)
             return None
 
         task = asyncio.create_task(func(*args, **kwargs))
@@ -323,12 +369,43 @@ def _task_done(self, task: asyncio.Task):
             self._record_error(exc)
 
     def _record_error(self, exc: BaseException):
-        self.last_error = f"{type(exc).__name__}: {exc}"
-        logger.error("twin component failed: %s", self.last_error, exc_info=exc)
+        """Route a failure into the twin state, and stop the twin.
+
+        A component failure is a twin failure: the other components have
+        lost the graph they were part of, so the twin is torn down exactly
+        as `stop()` would tear it down -- but it ends up `failed`, with
+        `last_error`, rather than `stopped`.
+
+        Called from several doors, all of them synchronous and all of them
+        on the host loop: task done-callbacks, the teardown itself, and
+        `PubSubClient.on_error` (the stream backends report from the
+        done-callback of their receive loop; a backend whose events arrive
+        on a foreign thread hands them over with `call_soon_threadsafe`
+        first).  Teardown is therefore *scheduled*, never awaited here.
+
+        Re-entrant by construction: the first failure owns the report and
+        the teardown, and everything after it is fallout -- teardown
+        cancelling the failure's siblings, a stop hook tripping over a
+        half-dead component -- which is logged and dropped.
+        """
+
+        error = f"{type(exc).__name__}: {exc}"
+        logger.error("twin component failed: %s", error, exc_info=exc)
+
+        if self.state is RuntimeState.FAILED:
+            # the cause is already recorded, and its teardown is running
+            return
+
+        self.last_error = error
 
         # a stopped twin stays stopped, but keeps the error for inspection
-        if self.state is not RuntimeState.STOPPED:
-            self.state = RuntimeState.FAILED
+        if self.state is RuntimeState.STOPPED:
+            return
+
+        self.state = RuntimeState.FAILED
+
+        if self._stop_task is None:
+            self._stop_task = self._start_teardown(STOP_TIMEOUT)
 
     def _api(self, ant: _AnnotatedComponent) -> RuntimeAPI:
         return RuntimeAPI(self, ant)
diff --git a/test/unit/test_runtime_stop.py b/test/unit/test_runtime_stop.py
index dff7b08..e1388b8 100644
--- a/test/unit/test_runtime_stop.py
+++ b/test/unit/test_runtime_stop.py
@@ -13,6 +13,7 @@
     RuntimeState,
     UtilityTask,
 )
+from digitaltwin.runtime import STOP_TIMEOUT
 
 TICK = DataType("tick")
 
@@ -49,6 +50,71 @@ async def main_loop(self, runtime, in_data):
         raise ValueError("component exploded")
 
 
+class BoomOnCue(UtilityTask):
+    """A component which fails only once the test says so -- long enough
+    for its siblings to have proven that they were running."""
+
+    def __init__(self, flow, cue: asyncio.Event, error="component exploded"):
+        super().__init__(flow)
+        self.cue = cue
+        self.error = error
+
+    async def main_loop(self, runtime, in_data):
+        await self.cue.wait()
+        raise ValueError(self.error)
+
+
+class BoomOnCancel(UtilityTask):
+    """A component which fails *while being torn down*: its cancellation
+    handler raises something that is not a CancelledError."""
+
+    async def main_loop(self, runtime, in_data):
+        try:
+            await asyncio.sleep(3600)
+        except asyncio.CancelledError:
+            raise ValueError("component exploded on cancel")
+
+
+class Publisher(UtilityTask):
+    """A persistent component publishing on the twin's stream, recording
+    what it published so a test can watch it stop."""
+
+    def __init__(self, flow, published: list):
+        super().__init__(flow)
+        self.published = published
+
+    async def main_loop(self, runtime, in_data):
+        while True:
+            await runtime.stream.publish(TICK, len(self.published))
+            self.published.append(len(self.published))
+            await asyncio.sleep(0.01)
+
+
+async def torn_down(runtime: DTRuntime, timeout: float = 5.0):
+    """Await the teardown the twin started on its own.
+
+    `_stop_task` is the handle a scheduled teardown leaves behind -- the
+    failure path cannot hand a caller anything else, since it is reached
+    from synchronous done-callbacks.
+    """
+
+    async def wait():
+        while runtime._stop_task is None:
+            await asyncio.sleep(0.01)
+        await asyncio.shield(runtime._stop_task)
+
+    await asyncio.wait_for(wait(), timeout=timeout)
+
+
+def assert_stream_is_closed(client):
+    """The leak assertion the explicit-stop tests use."""
+
+    assert client.subscriptions == set()
+    assert client._backend.pub_soc is None
+    assert client._backend.sub_soc is None
+    assert client._backend._ctx.closed
+
+
 async def test_stop_is_terminal_and_idempotent(stream_clients):
     runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
     runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
@@ -90,10 +156,7 @@ async def test_stop_cancels_tasks_and_tears_down_the_stream(
 
     # no lingering tasks, subscriptions, sockets or contexts
     assert runtime.running_tasks == set()
-    assert client.subscriptions == set()
-    assert client._backend.pub_soc is None
-    assert client._backend.sub_soc is None
-    assert client._backend._ctx.closed
+    assert_stream_is_closed(client)
 
 
 async def test_stop_abandons_tasks_that_ignore_cancellation(
@@ -146,10 +209,16 @@ async def test_graph_cannot_be_changed_after_stop(stream_clients):
         runtime.add_barrier(Barrier("b"))
 
 
-async def test_stream_failure_fails_the_twin(stream_clients):
+async def test_stream_failure_fails_the_twin(stream_clients, no_task_leaks):
+    """The out-of-band door: a failure only the host can see.  `on_error`
+    arrives synchronously, from the done-callback of the backend's receive
+    loop -- teardown can only be scheduled from there, never awaited."""
+
     client = await stream_clients("twin-a")
     runtime = DTRuntime(NO_FLOW, client)
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
     runtime.start()
+    await asyncio.sleep(0.1)
 
     # what a dead receive loop reports
     client._backend._report_error(RuntimeError("stream receive loop exited"))
@@ -157,25 +226,215 @@ async def test_stream_failure_fails_the_twin(stream_clients):
     assert runtime.state is RuntimeState.FAILED
     assert runtime.last_error == "RuntimeError: stream receive loop exited"
 
+    # ... and the twin took itself down over it
+    await torn_down(runtime)
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
     with pytest.raises(RuntimeError):
         runtime.start()
 
     await runtime.stop()
 
 
-async def test_component_failure_sets_failed_state(stream_clients):
-    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
-    runtime.add_task(Boom(NO_FLOW), TRUTHY, NULL_DTYPE)
+async def test_component_failure_tears_the_twin_down(stream_clients, no_task_leaks):
+    """A component failure stops the twin: same teardown as `stop()`, but
+    it ends in `failed` with the error, not in `stopped`."""
+
+    cue = asyncio.Event()
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+
+    barrier = Barrier("b")
+    barrier.add_dtype(TICK)
+    runtime.add_barrier(barrier)
+
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    runtime.add_task(BoomOnCue(NO_FLOW, cue), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    # the twin is up: tasks running, subscribed to its own output dtype
+    await asyncio.sleep(0.3)
+    assert runtime.running_tasks
+    assert TICK in client.subscriptions
+
+    cue.set()
+    await torn_down(runtime)
+
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "ValueError: component exploded"
+
+    # nothing of the twin is left running or open
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
+    # and it starts no further work
+    assert runtime._to_asyncio_task(asyncio.sleep, 0) is None
+    assert runtime.running_tasks == set()
+
+
+async def test_an_out_of_band_failure_tears_the_twin_down(
+    stream_clients, no_task_leaks
+):
+    """A failure the runtime cannot see by itself, handed in from outside
+    and synchronously -- what the service's `fail()` door reports when an
+    engine endpoint is lost under a twin (R8).  Same funnel, same
+    teardown: the twin is down when it says `failed`."""
+
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
     runtime.start()
 
     await asyncio.sleep(0.1)
+    assert runtime.running_tasks
+
+    runtime._record_error(RuntimeError("engine endpoint lost: hpc1"))
+
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "RuntimeError: engine endpoint lost: hpc1"
+
+    await torn_down(runtime)
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
+
+async def test_a_failure_stops_the_other_components(stream_clients, no_task_leaks):
+    """The failure stops the *twin*, not just the component which broke:
+    a persistent publisher which was happily publishing goes quiet."""
+
+    published: list[int] = []
+    cue = asyncio.Event()
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+
+    runtime.add_task(Publisher(NO_FLOW, published), TRUTHY, TICK, is_persistent=True)
+    runtime.add_task(BoomOnCue(NO_FLOW, cue), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    async def wait_for_data():
+        while len(published) < 3:
+            await asyncio.sleep(0.01)
+
+    await asyncio.wait_for(wait_for_data(), timeout=10.0)
+
+    cue.set()
+    await torn_down(runtime)
+
+    # whatever it had published by then, it publishes nothing more
+    quiet_at = len(published)
+    await asyncio.sleep(0.2)
+
+    assert published == list(range(quiet_at))
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
+
+async def test_stop_after_a_failure_is_a_bounded_no_op(stream_clients, no_task_leaks):
+    """`TwinInstance.close()` calls `stop()` on whatever it holds.  On an
+    already torn-down twin that returns promptly, does not raise, and
+    leaves the failure standing: the error is the more useful fact."""
+
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+    runtime.add_task(Boom(NO_FLOW), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    await torn_down(runtime)
+    teardown = runtime._stop_task
+
+    await asyncio.wait_for(runtime.stop(), timeout=2.0)
+    await asyncio.wait_for(runtime.stop(timeout=0.1), timeout=2.0)
+
+    # no second teardown, no state change, no clobbered error
+    assert runtime._stop_task is teardown
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "ValueError: component exploded"
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
+    with pytest.raises(RuntimeError):
+        runtime.start()
+
+
+async def test_stop_during_a_failure_teardown_joins_it(stream_clients, no_task_leaks):
+    """The race `TwinInstance.close()` can lose: a component fails while a
+    client is calling `stop()`.  One teardown, and the failure wins the
+    state -- `stop()` never gets to flip it to `stopped`."""
+
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+    runtime.add_task(Forever(NO_FLOW), TRUTHY, TICK, is_persistent=True)
+    runtime.add_task(Boom(NO_FLOW), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    # the failure is already scheduled, its teardown has not run yet
+    await asyncio.sleep(0)
+    await asyncio.sleep(0)
+    assert runtime.state is RuntimeState.FAILED
+
+    teardown = runtime._stop_task
+    await runtime.stop()
+
+    assert runtime._stop_task is teardown
     assert runtime.state is RuntimeState.FAILED
     assert runtime.last_error == "ValueError: component exploded"
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
 
-    # stop still works on a failed twin, and keeps the error
+
+async def test_a_failure_after_stop_keeps_the_twin_stopped(stream_clients):
+    """The mirror-image rule: a twin which was stopped cleanly stays
+    `stopped`, and only keeps the error for inspection."""
+
+    runtime = DTRuntime(NO_FLOW, await stream_clients("twin-a"))
+    runtime.start()
     await runtime.stop()
+
+    runtime._record_error(RuntimeError("late arrival"))
+
     assert runtime.state is RuntimeState.STOPPED
-    assert runtime.last_error == "ValueError: component exploded"
+    assert runtime.last_error == "RuntimeError: late arrival"
+
+
+async def test_simultaneous_failures_run_one_teardown(stream_clients, no_task_leaks):
+    """Two components failing in the same loop iteration, and a third
+    failing *inside* the teardown (its cancellation handler raises): one
+    teardown, no crash, and the first error is the one reported."""
+
+    client = await stream_clients("twin-a")
+    runtime = DTRuntime(NO_FLOW, client)
+
+    scheduled = []
+    start_teardown = runtime._start_teardown
+
+    def spy(timeout):
+        scheduled.append(timeout)
+        return start_teardown(timeout)
+
+    runtime._start_teardown = spy
+
+    cue = asyncio.Event()
+    cue.set()
+    runtime.add_task(BoomOnCue(NO_FLOW, cue, "first"), TRUTHY, NULL_DTYPE)
+    runtime.add_task(BoomOnCue(NO_FLOW, cue, "second"), TRUTHY, NULL_DTYPE)
+    runtime.add_task(BoomOnCancel(NO_FLOW), TRUTHY, NULL_DTYPE)
+    runtime.start()
+
+    await torn_down(runtime)
+
+    # one teardown, on the default budget
+    assert scheduled == [STOP_TIMEOUT]
+    assert runtime.state is RuntimeState.FAILED
+    assert runtime.last_error == "ValueError: first"
+    assert runtime.running_tasks == set()
+    assert_stream_is_closed(client)
+
+    await runtime.stop()
+    assert runtime.last_error == "ValueError: first"
 
 
 async def test_stopped_runtime_starts_no_new_work(stream_clients):

From 3fd563844926423a134bc7a2afb66e536b1ab7cd Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Mon, 17 Aug 2026 19:53:26 +0200
Subject: [PATCH 11/19] The graph opens at its input edge: external channels

Sensors are external entities.  A sensor is not a twin component: it is a
process with a lifetime of its own, it precedes and outlives any twin,
and one instrument's data is wanted by however many twins care about it.
Persistent components can produce data, but they cannot be how data
enters a graph.

`DTRuntime.add_input(dtype, channel, codec='json')` binds a shared
external channel to an input dtype.  The channel topic goes on the
backend verbatim, with no `dt/<namespace>` prefix, which is exactly what
makes it shareable: n twins may bind the same channel and the pubsub
fan-out gives each of them every message.  A channel claiming the
internal prefix is refused, as is an unknown codec, and both are refused
at registration rather than on the first message.  From the dtype queue
onwards, external data is indistinguishable from internal traffic, so
consumers, barriers and investigators need no notion of where it came
from.  `stop()` drops the bindings with every other subscription, and
`describe()` reports them.

What goes on the wire is a deployment decision, not ours: `json` for the
plain scripts and instruments which are the normal producers, `raw` for
bytes, `cloudpickle` only for producers inside the same trust domain.  An
undecodable payload costs its own message and is logged, the same
contract malformed internal traffic already had.

The codecs and the verbatim subscribe live at `PubSubClient` level, above
the backend seam.  A backend learns one new thing, through the
`backend_params` it already accepted: `raw` payloads are opaque bytes
which it hands over untouched, because something above owns their format.
Nothing zmq-specific moved upwards, and nothing above knows a topic is a
zmq subscription.

`ChannelPublisher` is the other half, for code which is not part of the
framework at all: a broker, a channel, a codec, no namespace and no
runtime.  `PubSubConfig` grew `connect_backend()` for it and its
namespace became optional, since a channel belongs to no twin.

Demos 01 and 04 now run their sensor as a separate process publishing
JSON on a shared channel, and their twins bind it with `add_input`.  The
timer-driven demos (06, 07) stay as they were and are documented as the
internal-producer examples.  The README explains why the sensor gets a
terminal of its own.

Tests: two twins on one channel both receive every message, raw
delivery, an undecodable payload dropped without stalling the stream,
channel and codec validation, bindings in `describe()`, and codec round
trips.  40 passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 README.md                              |  28 ++-
 src/digitaltwin/__init__.py            |   8 +
 src/digitaltwin/runtime.py             | 115 +++++++++---
 src/digitaltwin/streaming.py           | 235 +++++++++++++++++++++++--
 test/01-start-inference-stop/dtypes.py |   5 +
 test/01-start-inference-stop/run_me.py |  11 +-
 test/01-start-inference-stop/sensor.py |  46 +++--
 test/04-start-agent-stop/dtypes.py     |   3 +
 test/04-start-agent-stop/run_me.py     |  11 +-
 test/04-start-agent-stop/sensor.py     |  42 +++--
 test/06-agent-pi/README.md             |   9 +
 test/unit/test_inputs.py               | 184 +++++++++++++++++++
 12 files changed, 606 insertions(+), 91 deletions(-)
 create mode 100644 test/unit/test_inputs.py

diff --git a/README.md b/README.md
index 163bde8..6b2fcef 100644
--- a/README.md
+++ b/README.md
@@ -37,15 +37,31 @@ The unit tests start their own stream broker on a random port; no setup.
 3. In one terminal, run `local_broker.py` -- it prints the addresses it
    bound
 4. In a second terminal, cd into the demo and run `run_me.py`
+5. In a third terminal, in the same demo, run its `sensor.py` if it has
+   one (`01-start-inference-stop` and `04-start-agent-stop` do)
 
 **When running a demo: be sure to start the ZMQ PubSub broker!**
 
-Both sides resolve the broker addresses the same way: `DT_STREAM_PUB_ADDR`
+The third terminal is the point, not an inconvenience: a sensor is an
+external entity.  It is a process of its own with a lifetime of its own,
+it publishes JSON on a shared channel, and it knows nothing about twins.
+The twin binds that channel with `runtime.add_input(dtype, channel)`, and
+a second twin binding the same channel receives the same messages -- which
+is how one instrument feeds many twins.  Start and stop the sensor
+independently of the twin; neither cares.
+
+Demos without a `sensor.py` produce their input inside the twin, which is
+what persistent components are still for: `06-agent-pi` drives itself off
+a timer, and `07-barrier` off several.
+
+Every side resolves the broker addresses the same way: `DT_STREAM_PUB_ADDR`
 and `DT_STREAM_SUB_ADDR`, defaulting to `tcp://127.0.0.1:5000` and `:5001`
-(see `digitaltwin.config`).  Set them in both terminals to move the broker.
+(see `digitaltwin.config`).  Set them in every terminal to move the broker.
 
 **Binding policy**: the broker binds to loopback by default, and it must
-stay that way unless you know what you are doing -- pubsub payloads are
-cloudpickled, so anyone who can reach the broker ports can execute code in
-every subscriber.  A non-loopback bind needs an explicit configuration and
-a private/firewalled network.
+stay that way unless you know what you are doing -- twin-internal payloads
+are cloudpickled, so anyone who can reach the broker ports can execute code
+in every subscriber.  A non-loopback bind needs an explicit configuration
+and a private/firewalled network.  External channels are decoded with the
+codec their binding names: `json` (the default) and `raw` are safe to
+accept from a producer you do not control, `cloudpickle` is not.
diff --git a/src/digitaltwin/__init__.py b/src/digitaltwin/__init__.py
index e88a0a4..d882b6c 100644
--- a/src/digitaltwin/__init__.py
+++ b/src/digitaltwin/__init__.py
@@ -16,6 +16,10 @@
 from .config import stream_addresses
 from .runtime import DTRuntime, RuntimeAPI, RuntimeState
 from .streaming import (
+    CODEC_CLOUDPICKLE,
+    CODEC_JSON,
+    CODEC_RAW,
+    ChannelPublisher,
     PubSubBackend,
     PubSubClient,
     PubSubConfig,
@@ -26,9 +30,13 @@
 )
 
 __all__ = [
+    "CODEC_CLOUDPICKLE",
+    "CODEC_JSON",
+    "CODEC_RAW",
     "NULL_DTYPE",
     "TRUTHY",
     "Barrier",
+    "ChannelPublisher",
     "DTRuntime",
     "DataType",
     "ModelInvestigator",
diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index 93abdf8..e54a582 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -20,7 +20,7 @@
     UtilityTask,
     _TwinComponent,
 )
-from .streaming import PubSubClient, PubSubConfig
+from .streaming import CODEC_JSON, PubSubClient, PubSubConfig, check_codec
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +28,15 @@
 STOP_TIMEOUT = 10.0
 
 
+@dataclass(frozen=True)
+class _InputBinding:
+    """An external channel bound to an input dtype (see `add_input`)."""
+
+    dtype: DataType
+    channel: str
+    codec: str
+
+
 class RuntimeState(StrEnum):
     """Lifecycle of a twin runtime.
 
@@ -219,6 +228,9 @@ def __init__(self, flow: WorkflowEngine, streamer: PubSubClient):
         # list of barriers: order of PRODUCE --> CONSUME
         self.barriers: dict[DataType, list[Barrier]] = defaultdict(list)
 
+        # the graph's input edge: external channels feeding a dtype
+        self.inputs: list[_InputBinding] = []
+
         self.running_tasks: set[asyncio.Task] = set()
 
         self.is_start = asyncio.Event()
@@ -411,8 +423,60 @@ def _api(self, ant: _AnnotatedComponent) -> RuntimeAPI:
         return RuntimeAPI(self, ant)
 
     def _check_mutable(self):
-        if self.state is RuntimeState.STOPPED:
-            raise RuntimeError("twin is stopped - its graph cannot be changed")
+        # both terminal states are terminal for the graph as well
+        if self.state in (RuntimeState.STOPPED, RuntimeState.FAILED):
+            raise RuntimeError(f"twin is {self.state} - its graph cannot be changed")
+
+    def _ensure_dtype_queue(self, dtype: DataType) -> asyncio.Queue:
+        """The queue feeding the components registered for `dtype`, with
+        its consumer task running."""
+
+        if dtype not in self.dtype_queues:
+            logger.info(f"Create listener for: {dtype}")
+            self.dtype_queues[dtype] = asyncio.Queue()
+            self._to_asyncio_task(self._launch_consumer, dtype)
+
+        return self.dtype_queues[dtype]
+
+    def add_input(self, dtype: DataType, channel: str, codec: str = CODEC_JSON):
+        """Open the graph at its input edge: bind an external channel.
+
+        Sensors and other producers live outside the framework.  They
+        publish to a shared channel, this binds that channel to an input
+        dtype, and from there the data flows exactly like traffic a
+        component produced.  The channel topic is shared verbatim, so any
+        number of twins may bind the same one and each of them receives
+        every message on it.
+
+        `codec` decodes the payloads: `json` for the plain scripts and
+        instruments which are the normal producers, `raw` for bytes, and
+        `cloudpickle` only for producers inside the same trust domain
+        (see the binding policy in the README).
+
+        Internal producers keep their own path: a persistent component
+        publishes through `RuntimeAPI.stream`.
+        """
+
+        self._check_mutable()
+
+        PubSubClient.check_channel(channel)
+        check_codec(codec)
+
+        binding = _InputBinding(dtype, channel, codec)
+        if binding in self.inputs:
+            return
+        self.inputs.append(binding)
+
+        # subscribe now, so nothing published before start() is lost: the
+        # queue buffers it and the consumers wait for start anyway
+        logger.info(f"Bind channel {channel!r} ({codec}) to dtype: {dtype}")
+        self._to_asyncio_task(
+            self.streamer.subscribe_to_channel,
+            channel,
+            dtype,
+            self._ensure_dtype_queue(dtype),
+            codec,
+        )
 
     def add_task(
         self,
@@ -585,14 +649,9 @@ async def _run_component(
                     ant.output_dtype in self.components
                     or ant.output_dtype in self.barriers
                 ):
-                    # has a task registered, but no queue yet.
-                    if ant.output_dtype not in self.dtype_queues:
-                        self.dtype_queues[ant.output_dtype] = asyncio.Queue()
-                        self._to_asyncio_task(self._launch_consumer, ant.output_dtype)
-
                     logger.info(f"Subscribe to dtype: {ant.output_dtype}")
                     await self.streamer.subscribe_to_dtype(
-                        ant.output_dtype, self.dtype_queues[ant.output_dtype]
+                        ant.output_dtype, self._ensure_dtype_queue(ant.output_dtype)
                     )
                 # else: output is null.
 
@@ -708,24 +767,13 @@ def _put_to_dtype_queue(self, t_data: TypedData):
         if t_data.dtype == NULL_DTYPE:
             return
 
-        if t_data.dtype in self.dtype_queues:
-            logger.info(f"Enqueue: {t_data.dtype}")
-            self.dtype_queues[t_data.dtype].put_nowait(t_data)
-            return
-
-        # if not in there, check if there are any associated tasks.
-        # if not, drop the dtype
-        if t_data.dtype not in self.components:
-            return
-
-        # has a task registered, but no queue yet.
-        self.dtype_queues[t_data.dtype] = asyncio.Queue()
-        self.dtype_queues[t_data.dtype].put_nowait(t_data)
+        # nothing registered for it, and nothing consuming it: drop it
+        if t_data.dtype not in self.dtype_queues:
+            if t_data.dtype not in self.components:
+                return
 
-        # create consumer task
-        logger.info(f"Create listener for: {t_data.dtype}")
         logger.info(f"Enqueue: {t_data.dtype}")
-        self._to_asyncio_task(self._launch_consumer, t_data.dtype)
+        self._ensure_dtype_queue(t_data.dtype).put_nowait(t_data)
 
     async def _launch_b_consumer(self, dtype: DataType, creation: asyncio.Event):
         await creation.wait()
@@ -779,14 +827,25 @@ def described(ant: _AnnotatedComponent) -> dict:
             described(ant) for ants in self.components.values() for ant in ants
         ]
 
+        inputs = [
+            {
+                "dtype": binding.dtype.name,
+                "channel": binding.channel,
+                "codec": binding.codec,
+            }
+            for binding in self.inputs
+        ]
+
         return {
             "namespace": self.streamer.namespace,
             "state": str(self.state),
             "last_error": self.last_error,
+            "inputs": inputs,
             "components": components,
             "dtypes": sorted(
                 {entry["input_dtype"] for entry in components}
                 | {entry["output_dtype"] for entry in components}
+                | {entry["dtype"] for entry in inputs}
             ),
             # per dtype, the ordered chain of barriers it passes through
             "barriers": {
@@ -809,6 +868,12 @@ def print_graph(self) -> str:
 
         lines = ["=" * 30, f"Digital Twin Flow: {info['namespace']} [{info['state']}]"]
 
+        for binding in info["inputs"]:
+            lines.append(
+                f"CHANNEL: {binding['channel']} ({binding['codec']})"
+                f" --> {binding['dtype']}"
+            )
+
         for input_dtype, entries in by_input.items():
             lines.append(f"IN: {input_dtype}")
             for entry in entries:
diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index 2eb8233..7260aa4 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -4,6 +4,7 @@
 
 import asyncio
 import contextlib
+import json
 import logging
 import multiprocessing
 
@@ -22,6 +23,48 @@
 
 logger = logging.getLogger(__name__)
 
+# Payload codecs for external channels.  A channel is shared with
+# producers which are not part of this framework, so what goes on the wire
+# is a deployment decision, not ours: plain instruments and scripts speak
+# `json`, `raw` hands bytes through untouched, and `cloudpickle` is only
+# for producers inside the same trust domain (it executes what it decodes,
+# see the binding policy in the README).
+CODEC_JSON = "json"
+CODEC_RAW = "raw"
+CODEC_CLOUDPICKLE = "cloudpickle"
+
+_CODECS: dict[str, tuple[Callable, Callable]] = {
+    CODEC_JSON: (
+        lambda message: json.dumps(message).encode("utf-8"),
+        lambda payload: json.loads(payload.decode("utf-8")),
+    ),
+    CODEC_RAW: (bytes, lambda payload: payload),
+    CODEC_CLOUDPICKLE: (cloudpickle.dumps, cloudpickle.loads),
+}
+
+
+def check_codec(codec: str):
+    """Reject an unknown codec name, at registration rather than on the
+    first message."""
+
+    if codec not in _CODECS:
+        raise ValueError(
+            f"unknown stream codec: {codec!r}" f" (known: {', '.join(sorted(_CODECS))})"
+        )
+
+
+def encode_payload(message, codec: str) -> bytes:
+    check_codec(codec)
+
+    return _CODECS[codec][0](message)
+
+
+def decode_payload(payload: bytes, codec: str):
+    check_codec(codec)
+
+    return _CODECS[codec][1](payload)
+
+
 # bounded waits: no teardown path may hang the host event loop
 BROKER_START_TIMEOUT = 30.0
 BROKER_STOP_TIMEOUT = 5.0
@@ -56,12 +99,22 @@ async def connect(self, *args, **kwargs):
         pass
 
     @abstractmethod
-    async def publish(self, topic, message, **kwargs):
-        pass
+    async def publish(self, topic, message, raw=False, **kwargs):
+        """Publish `message` on `topic`.
+
+        `raw` means the payload is already bytes and something above the
+        seam owns its wire format: hand it over untouched.  Channels use
+        that (see the codecs); twin-internal traffic does not, and the
+        backend serializes it however it likes.
+        """
 
     @abstractmethod
-    async def subscribe(self, topic, callback, **kwargs):
-        pass
+    async def subscribe(self, topic, callback, raw=False, **kwargs):
+        """Deliver messages published on `topic` to `callback`.
+
+        With `raw`, the callback receives the payload bytes as they
+        arrived, undecoded.
+        """
 
     @abstractmethod
     async def unsubscribe(self, topic):
@@ -256,6 +309,10 @@ def __init__(self, pub_addr: Optional[str] = None, sub_addr: Optional[str] = Non
 
         self.topics: dict[str, list[Callable]] = {}
 
+        # topics whose payload the transport must hand over untouched:
+        # something above the seam owns their wire format (see the codecs)
+        self.raw_topics: set[str] = set()
+
         self._task: Optional[asyncio.Task] = None
         self._closed = False
         self.is_running = asyncio.Event()
@@ -296,7 +353,7 @@ async def connect(self):
 
         self.is_running.set()
 
-    async def publish(self, topic, message):
+    async def publish(self, topic, message, raw=False):
         self._check_open()
         if self.pub_soc is None:
             raise ValueError("Publishing endpoint not connected")
@@ -306,10 +363,10 @@ async def publish(self, topic, message):
             await self.is_running.wait()
 
         topic_b = topic.encode("utf-8")
-        message_b = cloudpickle.dumps(message)
+        message_b = message if raw else cloudpickle.dumps(message)
         await self.pub_soc.send_multipart([topic_b, message_b])
 
-    async def subscribe(self, topic, callback, **backend_params):
+    async def subscribe(self, topic, callback, raw=False, **backend_params):
         self._check_open()
         if self.sub_soc is None:
             raise ValueError("Subscribe endpoint not connected")
@@ -320,6 +377,8 @@ async def subscribe(self, topic, callback, **backend_params):
         self.sub_soc.setsockopt(zmq.SUBSCRIBE, topic.encode("utf-8"))
 
         self.topics.setdefault(topic, []).append(callback)
+        if raw:
+            self.raw_topics.add(topic)
 
     async def unsubscribe(self, topic):
         if self._closed or self.sub_soc is None:
@@ -328,6 +387,7 @@ async def unsubscribe(self, topic):
         if topic in self.topics:
             self.sub_soc.setsockopt(zmq.UNSUBSCRIBE, topic.encode("utf-8"))
             del self.topics[topic]
+            self.raw_topics.discard(topic)
 
     async def close(self):
         """Cancel the receive loop, close all sockets, terminate the context.
@@ -349,6 +409,7 @@ async def close(self):
             # the guard above makes close() a one-shot, so the sockets and
             # the context have to go even if the await was cancelled
             self.topics.clear()
+            self.raw_topics.clear()
 
             for sock in (self.pub_soc, self.sub_soc):
                 if sock is not None:
@@ -373,7 +434,9 @@ async def _run(self):
             try:
                 topic, message = frames
                 item = topic.decode("utf-8")
-                data = cloudpickle.loads(message)
+                data = (
+                    message if item in self.raw_topics else cloudpickle.loads(message)
+                )
             except Exception:
                 logger.exception("dropping malformed message: %r", frames[:1])
                 continue
@@ -417,6 +480,10 @@ class PubSubClient:
     # used because a dtype label may contain any printable character.
     TOPIC_TERMINATOR = "\x00"
 
+    # twin-internal traffic lives under this prefix; external channels are
+    # deliberately outside it, and may not claim it
+    TOPIC_PREFIX = "dt/"
+
     # For now, only support one backend. Future TODO: Add support for multiple backends
 
     def __init__(self, backend: PubSubBackend, namespace: str):
@@ -431,6 +498,9 @@ def __init__(self, backend: PubSubBackend, namespace: str):
         # so I don't repeat
         self.subscriptions: set[DataType] = set()
 
+        # external channels bound to a dtype: (channel, dtype)
+        self.channels: set[tuple[str, DataType]] = set()
+
     @property
     def on_error(self) -> Optional[Callable[[BaseException], None]]:
         """Hook for asynchronous stream failures (see `PubSubBackend`)."""
@@ -453,7 +523,80 @@ def config(self) -> "PubSubConfig":
         )
 
     def topic(self, dtype: DataType) -> str:
-        return f"dt/{self.namespace}/dtypes/{dtype.name}{self.TOPIC_TERMINATOR}"
+        """Topic carrying this twin's internal traffic for `dtype`."""
+
+        return (
+            f"{self.TOPIC_PREFIX}{self.namespace}/dtypes/{dtype.name}"
+            f"{self.TOPIC_TERMINATOR}"
+        )
+
+    @classmethod
+    def check_channel(cls, channel: str):
+        """Reject a channel name which is not usable as an external topic.
+
+        A channel goes on the wire verbatim, which is what makes it
+        shareable.  It may therefore not claim the prefix under which twins
+        publish their internal traffic, and it may not carry the topic
+        terminator.
+        """
+
+        if not channel:
+            raise ValueError("a channel name is required")
+
+        if channel.startswith(cls.TOPIC_PREFIX):
+            raise ValueError(
+                f"channel {channel!r} collides with twin-internal topics"
+                f" ({cls.TOPIC_PREFIX}...): pick a name outside that prefix"
+            )
+
+        if cls.TOPIC_TERMINATOR in channel:
+            raise ValueError(f"channel {channel!r} contains a topic terminator")
+
+    async def subscribe_to_channel(
+        self,
+        channel: str,
+        dtype: DataType,
+        queue: asyncio.Queue,
+        codec: str = CODEC_JSON,
+        backend_params={},
+    ):
+        """Feed an external channel into `dtype`.
+
+        The topic is used verbatim, without this twin's namespace: the
+        channel is shared, so every twin which binds it receives every
+        message on it, and the producers are outside the framework
+        entirely.  Payloads are decoded per `codec`; an undecodable one is
+        dropped and logged rather than taking the stream down with it.
+        """
+
+        self.check_channel(channel)
+        check_codec(codec)
+
+        if (channel, dtype) in self.channels:
+            return
+        self.channels.add((channel, dtype))
+
+        async def receive_data(payload):
+            try:
+                message = decode_payload(payload, codec)
+            except Exception:
+                logger.exception(
+                    "dropping undecodable %s payload on channel %r", codec, channel
+                )
+                return
+
+            await queue.put(TypedData(dtype, message))
+
+        await self._backend.subscribe(
+            topic=channel, callback=receive_data, raw=True, **backend_params
+        )
+
+    async def unsubscribe_channel(self, channel: str, dtype: DataType):
+        if (channel, dtype) not in self.channels:
+            return
+        self.channels.discard((channel, dtype))
+
+        await self._backend.unsubscribe(channel)
 
     # for runtime use only!!!
     async def subscribe_to_dtype(
@@ -498,6 +641,9 @@ async def close(self):
         for dtype in list(self.subscriptions):
             await self.unsubscribe_dtype(dtype)
 
+        for channel, dtype in list(self.channels):
+            await self.unsubscribe_channel(channel, dtype)
+
         await self._backend.close()
 
 
@@ -522,7 +668,7 @@ class PubSubConfig:
     further backend needs no change here.
     """
 
-    namespace: str
+    namespace: Optional[str] = None
     pub_addr: Optional[str] = None
     sub_addr: Optional[str] = None
     kind: str = ZMQ_PS_Client.kind
@@ -530,7 +676,7 @@ class PubSubConfig:
     @classmethod
     def resolve(
         cls,
-        namespace: str,
+        namespace: Optional[str] = None,
         pub_addr: Optional[str] = None,
         sub_addr: Optional[str] = None,
     ) -> "PubSubConfig":
@@ -539,11 +685,14 @@ def resolve(
 
         return cls(namespace, *stream_addresses(pub_addr, sub_addr))
 
-    async def connect(self, timeout: Optional[float] = None) -> PubSubClient:
-        """Open a connected, namespaced client for this endpoint.
+    async def connect_backend(self, timeout: Optional[float] = None) -> PubSubBackend:
+        """Open the transport alone, without namespace semantics.
+
+        What an external producer needs (see `ChannelPublisher`): a channel
+        is shared, so it belongs to no twin's namespace.
 
-        Bounded by `timeout` (None waits forever).  A client which fails to
-        connect is closed before the error propagates, so an unreachable
+        Bounded by `timeout` (None waits forever).  A backend which fails
+        to connect is closed before the error propagates, so an unreachable
         broker leaks neither sockets nor a context.
         """
 
@@ -560,7 +709,61 @@ async def connect(self, timeout: Optional[float] = None) -> PubSubClient:
             await backend.close()
             raise
 
-        return PubSubClient(backend, self.namespace)
+        return backend
+
+    async def connect(self, timeout: Optional[float] = None) -> PubSubClient:
+        """Open a connected, namespaced client for this endpoint."""
+
+        return PubSubClient(await self.connect_backend(timeout), self.namespace)
+
+
+class ChannelPublisher:
+    """Publishes to a shared channel from outside the framework.
+
+    Sensors and instruments are external entities: they are not twin
+    components, they outlive and precede any twin, and one channel of
+    theirs feeds however many twins bind it.  Such a producer needs a
+    broker, a channel name and a codec, and nothing else from this package
+    -- no namespace (a channel has none) and no runtime.
+    """
+
+    def __init__(self, backend: PubSubBackend, channel: str, codec: str = CODEC_JSON):
+        PubSubClient.check_channel(channel)
+        check_codec(codec)
+
+        self._backend = backend
+        self.channel = channel
+        self.codec = codec
+
+    @classmethod
+    async def open(
+        cls,
+        channel: str,
+        codec: str = CODEC_JSON,
+        config: Optional[PubSubConfig] = None,
+        timeout: Optional[float] = None,
+    ) -> "ChannelPublisher":
+        """Connect to a broker and publish to `channel` on it.
+
+        `config` defaults to the configured broker (environment, else the
+        loopback defaults); its namespace, if it has one, is ignored.
+        """
+
+        config = config or PubSubConfig.resolve()
+
+        return cls(await config.connect_backend(timeout), channel, codec)
+
+    async def publish(self, message):
+        """Publish one codec-encoded message on the channel."""
+
+        await self._backend.publish(
+            topic=self.channel,
+            message=encode_payload(message, self.codec),
+            raw=True,
+        )
+
+    async def close(self):
+        await self._backend.close()
 
 
 async def connect_stream_client(
diff --git a/test/01-start-inference-stop/dtypes.py b/test/01-start-inference-stop/dtypes.py
index ca16e88..0d7eb34 100644
--- a/test/01-start-inference-stop/dtypes.py
+++ b/test/01-start-inference-stop/dtypes.py
@@ -2,3 +2,8 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
+
+# the shared channel the external sensor publishes to.  It carries no twin
+# namespace: any number of twins may bind it and all of them see every
+# message.
+SENSOR_CHANNEL = "sensors/latency-probe"
diff --git a/test/01-start-inference-stop/run_me.py b/test/01-start-inference-stop/run_me.py
index ebecb1e..a2ec25f 100644
--- a/test/01-start-inference-stop/run_me.py
+++ b/test/01-start-inference-stop/run_me.py
@@ -5,10 +5,9 @@
 
 from digitaltwin.runtime import DTRuntime
 from digitaltwin.streaming import connect_stream_client
-from digitaltwin.components import TRUTHY, NULL_DTYPE
+from digitaltwin.components import NULL_DTYPE
 
 from dtypes import *
-from sensor import MySensor
 from model import MyModel
 from data_sink import MySink
 
@@ -18,7 +17,9 @@
 logger = logging.getLogger(__name__)
 
 # put it all together
-# sensor --> model --> data_sink
+# sensor channel --> model --> data_sink
+#
+# The sensor is external: run sensor.py in its own terminal.
 
 
 async def main():
@@ -36,11 +37,11 @@ async def main():
     runtime = DTRuntime(flow, pubsub_client)
 
     # create tasks and investigators
-    sensor = MySensor(flow)
     model = MyModel(flow)
     data_sink = MySink(flow)
 
-    runtime.add_task(sensor, TRUTHY, SENSOR_DTYPE, is_persistent=True)
+    # the graph opens at its input edge: bind the sensor's shared channel
+    runtime.add_input(SENSOR_DTYPE, SENSOR_CHANNEL)
     runtime.add_investigator(model, SENSOR_DTYPE, INFERENCE_DTYPE)
     runtime.add_task(data_sink, INFERENCE_DTYPE, NULL_DTYPE)
 
diff --git a/test/01-start-inference-stop/sensor.py b/test/01-start-inference-stop/sensor.py
index 52ffe9c..381640e 100644
--- a/test/01-start-inference-stop/sensor.py
+++ b/test/01-start-inference-stop/sensor.py
@@ -1,27 +1,37 @@
+"""The sensor, as an external entity: its own process, its own lifetime.
+
+Run it in a terminal of its own, once the broker is up:
+
+    python sensor.py
+
+It publishes JSON on a shared channel and knows nothing about twins.  The
+twin binds that channel with `runtime.add_input()`, and a second twin
+binding the same channel would receive the same messages.
+"""
+
 import asyncio
-import os
-import sys
 import time
 
-from radical.asyncflow import WorkflowEngine
-from digitaltwin.components import UtilityTask
-from dtypes import *
-import random
-
-import logging
+from digitaltwin.streaming import ChannelPublisher
 
-logger = logging.getLogger(__name__)
+from dtypes import SENSOR_CHANNEL
 
 
-class MySensor(UtilityTask):
-    def __init__(self, flow: WorkflowEngine):
-        super().__init__(flow)
-        self.flow = flow
+async def main():
+    publisher = await ChannelPublisher.open(SENSOR_CHANNEL)
 
-    async def main_loop(self, runtime, in_data):
+    try:
         for i in range(30):
-            # val = random.random()
-            val = time.monotonic_ns()
-            print(f"Sensor val: {val} - {i}")
-            await runtime.stream.publish(SENSOR_DTYPE, val)
+            # what the sink measures the end-to-end latency against
+            value = time.monotonic_ns()
+            print(f"Sensor val: {value} - {i}")
+
+            await publisher.publish(value)
             await asyncio.sleep(0.5)
+
+    finally:
+        await publisher.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/test/04-start-agent-stop/dtypes.py b/test/04-start-agent-stop/dtypes.py
index ca16e88..9999c7d 100644
--- a/test/04-start-agent-stop/dtypes.py
+++ b/test/04-start-agent-stop/dtypes.py
@@ -2,3 +2,6 @@
 
 SENSOR_DTYPE = DataType("sensor")
 INFERENCE_DTYPE = DataType("inference-from-mymodel")
+
+# the shared channel the external sensor publishes to
+SENSOR_CHANNEL = "sensors/random-walk"
diff --git a/test/04-start-agent-stop/run_me.py b/test/04-start-agent-stop/run_me.py
index 836b0e1..cda5373 100644
--- a/test/04-start-agent-stop/run_me.py
+++ b/test/04-start-agent-stop/run_me.py
@@ -5,10 +5,9 @@
 
 from digitaltwin.runtime import DTRuntime
 from digitaltwin.streaming import connect_stream_client
-from digitaltwin.components import TRUTHY, NULL_DTYPE
+from digitaltwin.components import NULL_DTYPE
 
 from dtypes import *
-from sensor import MySensor
 from agent import MyAgent
 from data_sink import MySink
 
@@ -18,7 +17,9 @@
 logger = logging.getLogger(__name__)
 
 # put it all together
-# sensor --> model --> data_sink
+# sensor channel --> agent --> data_sink
+#
+# The sensor is external: run sensor.py in its own terminal.
 
 
 async def main():
@@ -36,11 +37,11 @@ async def main():
     runtime = DTRuntime(flow, pubsub_client)
 
     # create tasks and investigators
-    sensor = MySensor(flow)
     agent = MyAgent(flow)
     data_sink = MySink(flow)
 
-    runtime.add_task(sensor, TRUTHY, SENSOR_DTYPE, is_persistent=True)
+    # the graph opens at its input edge: bind the sensor's shared channel
+    runtime.add_input(SENSOR_DTYPE, SENSOR_CHANNEL)
     runtime.add_agent(agent, SENSOR_DTYPE, INFERENCE_DTYPE)
     runtime.add_task(data_sink, INFERENCE_DTYPE, NULL_DTYPE)
 
diff --git a/test/04-start-agent-stop/sensor.py b/test/04-start-agent-stop/sensor.py
index 74189a7..375309a 100644
--- a/test/04-start-agent-stop/sensor.py
+++ b/test/04-start-agent-stop/sensor.py
@@ -1,25 +1,35 @@
-import asyncio
-import os
-import sys
+"""The sensor, as an external entity: its own process, its own lifetime.
+
+Run it in a terminal of its own, once the broker is up:
+
+    python sensor.py
 
-from radical.asyncflow import WorkflowEngine
-from digitaltwin.components import UtilityTask
-from dtypes import *
+It publishes JSON on a shared channel and knows nothing about twins.
+"""
+
+import asyncio
 import random
 
-import logging
+from digitaltwin.streaming import ChannelPublisher
 
-logger = logging.getLogger(__name__)
+from dtypes import SENSOR_CHANNEL
 
 
-class MySensor(UtilityTask):
-    def __init__(self, flow: WorkflowEngine):
-        super().__init__(flow)
-        self.flow = flow
+async def main():
+    publisher = await ChannelPublisher.open(SENSOR_CHANNEL)
 
-    async def main_loop(self, runtime, in_data):
+    try:
         while True:
             await asyncio.sleep(1)
-            val = random.random()
-            print(f"Sensor val: {val}")
-            await runtime.stream.publish(SENSOR_DTYPE, val)
+
+            value = random.random()
+            print(f"Sensor val: {value}")
+
+            await publisher.publish(value)
+
+    finally:
+        await publisher.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/test/06-agent-pi/README.md b/test/06-agent-pi/README.md
index 9aa09a9..a0d80b2 100644
--- a/test/06-agent-pi/README.md
+++ b/test/06-agent-pi/README.md
@@ -1,5 +1,14 @@
 
 
+# The internal-producer example
+
+This demo has no external sensor and no `sensor.py`: nothing outside
+measures anything.  Its input is a timer, i.e. a producer which belongs to
+the twin itself, so it stays a persistent component publishing through
+`RuntimeAPI.stream`.  Demos whose input comes from outside (01, 04) bind a
+shared channel with `runtime.add_input()` instead and run their sensor as
+a separate process.
+
 # Overview: calculating pi:
 
 1. Gregory-Leibniz series:
diff --git a/test/unit/test_inputs.py b/test/unit/test_inputs.py
new file mode 100644
index 0000000..b0ca400
--- /dev/null
+++ b/test/unit/test_inputs.py
@@ -0,0 +1,184 @@
+"""M0.7 -- the graph's input edge: external channels, shared by twins."""
+
+import asyncio
+
+import pytest
+
+from digitaltwin import (
+    CODEC_CLOUDPICKLE,
+    CODEC_JSON,
+    CODEC_RAW,
+    NULL_DTYPE,
+    ChannelPublisher,
+    DTRuntime,
+    DataType,
+    PubSubConfig,
+    UtilityTask,
+)
+from digitaltwin.streaming import decode_payload, encode_payload
+
+SENSOR = DataType("sensor")
+CHANNEL = "sensors/temperature"
+
+
+class Sink(UtilityTask):
+    def __init__(self, flow, received):
+        super().__init__(flow)
+        self.received = received
+
+    async def main_loop(self, runtime, in_data):
+        self.received.append(in_data.data)
+
+
+async def _publisher(broker, channel=CHANNEL, codec=CODEC_JSON):
+    """An external producer, as a demo script would open one."""
+
+    config = PubSubConfig(None, *broker.get_connection_str())
+
+    return await ChannelPublisher.open(channel, codec, config=config, timeout=10.0)
+
+
+async def _twin(stream_clients, namespace, channel=CHANNEL, codec=CODEC_JSON):
+    received: list = []
+
+    runtime = DTRuntime(None, await stream_clients(namespace))
+    runtime.add_input(SENSOR, channel, codec)
+    runtime.add_task(Sink(None, received), SENSOR, NULL_DTYPE)
+    runtime.start()
+
+    return runtime, received
+
+
+async def _wait_for(received, count, timeout=10.0):
+    async def wait():
+        while len(received) < count:
+            await asyncio.sleep(0.05)
+
+    await asyncio.wait_for(wait(), timeout)
+
+
+async def test_two_twins_share_one_channel(broker, stream_clients, no_task_leaks):
+    """The acceptance test for sharing: one external producer, one channel,
+    two twins in two namespaces, and every message reaches both."""
+
+    twin_a, received_a = await _twin(stream_clients, "twin-a")
+    twin_b, received_b = await _twin(stream_clients, "twin-b")
+
+    publisher = await _publisher(broker)
+    await asyncio.sleep(0.3)  # let both subscriptions reach the broker
+
+    try:
+        for value in range(5):
+            await publisher.publish({"value": value})
+
+        await _wait_for(received_a, 5)
+        await _wait_for(received_b, 5)
+
+        expected = [{"value": value} for value in range(5)]
+        assert received_a == expected
+        assert received_b == expected
+
+    finally:
+        await publisher.close()
+        await twin_a.stop()
+        await twin_b.stop()
+
+
+async def test_raw_channel_delivers_bytes(broker, stream_clients, no_task_leaks):
+    twin, received = await _twin(stream_clients, "twin-a", codec=CODEC_RAW)
+
+    publisher = await _publisher(broker, codec=CODEC_RAW)
+    await asyncio.sleep(0.3)
+
+    try:
+        await publisher.publish(b"\x01\x02 not text")
+        await _wait_for(received, 1)
+        assert received == [b"\x01\x02 not text"]
+
+    finally:
+        await publisher.close()
+        await twin.stop()
+
+
+async def test_undecodable_payload_is_dropped(broker, stream_clients, no_task_leaks):
+    """A producer which gets the codec wrong costs its own message, not the
+    stream (the same contract malformed internal traffic gets)."""
+
+    twin, received = await _twin(stream_clients, "twin-a")
+
+    # same channel, but bytes which are not JSON
+    garbage = await _publisher(broker, codec=CODEC_RAW)
+    publisher = await _publisher(broker)
+    await asyncio.sleep(0.3)
+
+    try:
+        await garbage.publish(b"\xff\xfe not json")
+        await asyncio.sleep(0.2)
+        assert received == []
+
+        await publisher.publish({"value": 1})
+        await _wait_for(received, 1)
+        assert received == [{"value": 1}]
+
+    finally:
+        await garbage.close()
+        await publisher.close()
+        await twin.stop()
+
+
+async def test_channels_and_codecs_are_validated(stream_clients):
+    runtime = DTRuntime(None, await stream_clients("twin-a"))
+
+    # a channel may not claim the twin-internal prefix, be empty, or carry
+    # the topic terminator
+    for channel in ("dt/twin-b/dtypes/sensor", "", "sensors/x\x00"):
+        with pytest.raises(ValueError):
+            runtime.add_input(SENSOR, channel)
+
+    with pytest.raises(ValueError):
+        runtime.add_input(SENSOR, CHANNEL, codec="yaml")
+
+    assert runtime.inputs == []
+
+    await runtime.stop()
+
+    # and a stopped twin takes no new bindings
+    with pytest.raises(RuntimeError):
+        runtime.add_input(SENSOR, CHANNEL)
+
+
+async def test_bindings_are_described(stream_clients):
+    import json
+
+    runtime = DTRuntime(None, await stream_clients("twin-a"))
+    runtime.add_input(SENSOR, CHANNEL)
+    runtime.add_input(SENSOR, CHANNEL)  # idempotent
+
+    info = runtime.describe()
+    assert json.loads(json.dumps(info)) == info
+    assert info["inputs"] == [{"dtype": "sensor", "channel": CHANNEL, "codec": "json"}]
+
+    # a bound dtype belongs to the graph even before a component consumes it
+    assert "sensor" in info["dtypes"]
+    assert CHANNEL in runtime.print_graph()
+
+    await runtime.stop()
+
+
+@pytest.mark.parametrize(
+    "codec, message",
+    [
+        (CODEC_JSON, {"value": 1.5, "unit": "C"}),
+        (CODEC_RAW, b"bytes"),
+        (CODEC_CLOUDPICKLE, {"value": 1.5}),
+    ],
+)
+def test_codecs_round_trip(codec, message):
+    assert decode_payload(encode_payload(message, codec), codec) == message
+
+
+def test_unknown_codec_is_refused():
+    with pytest.raises(ValueError):
+        encode_payload({}, "yaml")
+    with pytest.raises(ValueError):
+        decode_payload(b"{}", "yaml")

From 09eb2bac3f2011a2b69d70f802722baca607a4eb Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Mon, 17 Aug 2026 19:53:26 +0200
Subject: [PATCH 12/19] Plan: sensors are external, add_input is the entry path
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Records the revision agreed with the user in §1, adds the input-binding
item to M0 (item 7) and the verb to the §3 route list, and defers a
channel registry in §5.  Persistent tasks keep their place as internal
producers.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/dtaas-v1-plan.md | 45 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 5 deletions(-)

diff --git a/docs/dtaas-v1-plan.md b/docs/dtaas-v1-plan.md
index 42afe34..213b70f 100644
--- a/docs/dtaas-v1-plan.md
+++ b/docs/dtaas-v1-plan.md
@@ -53,9 +53,22 @@ Agreed semantics (decisions, not open questions):
   a co-located endpoint (same node as the broker) is the "local"
   deployment. The broker process runs only the DT control plane. No
   in-process `ProcessPoolExecutor` for user tasks.
-- **Persistent DT tasks (sensors, in-situ loops) run in the plugin
-  host process in v1** — as plain async `main_loop` code on the host
-  loop, using an injected, namespaced stream client (M0.3).
+- **Sensors are external entities.** The graph opens at its input
+  edge. A producer runs outside the framework and publishes to a
+  shared channel; a twin binds that channel to an input dtype with
+  `DTRuntime.add_input(dtype, channel, codec)` (M0.7). Channel topics
+  carry no twin namespace, so n twins may consume one channel and the
+  pubsub fan-out does the sharing. Producers precede and outlive any
+  twin, and neither side manages the other. Payload codecs are a
+  deployment choice: `json` for the plain scripts and instruments
+  which are the normal producers, `raw` for bytes, `cloudpickle` only
+  inside one trust domain (risk R7).
+- **Persistent DT tasks (internal producers, in-situ loops) run in
+  the plugin host process in v1** — as plain async `main_loop` code
+  on the host loop, using an injected, namespaced stream client
+  (M0.3). Internal producers are timers, agent loops and other
+  sources a twin owns. They are no longer how data enters a graph,
+  which is `add_input`.
   Persistent bodies are NOT `@flow.function_task`s: under an
   Orbit-backed engine a function task would be cloudpickled to the
   endpoint and occupy a backend slot for the twin's lifetime. This
@@ -301,9 +314,11 @@ limited to prerequisites P1-P2.
   bounded timeout.
 - Routes (per-session): `twin_create/{sid}`, `twin_list/{sid}`,
   `twin_close/{sid}/{twin_id}`, `twin_call/{sid}/{twin_id}` carrying
-  exactly one graph verb per request (`add_task`,
+  exactly one graph verb per request (`add_input`, `add_task`,
   `add_investigator`, `add_agent`, `start`, `stop`, `describe`,
-  `get_inference`; `describe` returns the serializable graph dict —
+  `get_inference`; `add_input` carries a dtype, a channel string and
+  a codec name, so it ships no artifact at all; `describe` returns
+  the serializable graph dict —
   human-readable rendering is a client concern). Barriers are
   **local-API only in v1**: `add_barrier` is not a remote verb — the
   `Barrier` object doesn't fit the `package()` class-shipping
@@ -390,6 +405,20 @@ plan; see status header.)
    **leak assertion**: no lingering tasks/sockets/contexts after
    twin teardown), namespacing, injected stream client; migrated
    demos.
+7. **Input bindings**: `DTRuntime.add_input(dtype, channel,
+   codec='json')` binds an external, shared channel to an input
+   dtype. The channel topic goes on the backend verbatim, which is
+   what makes it shareable, and a channel claiming the internal `dt/`
+   prefix is rejected. Decoded messages enter the dtype queue exactly
+   like internal traffic. Bindings are dropped by `stop()` with every
+   other subscription, and `describe()` reports them. Codecs and the
+   verbatim subscribe live at `PubSubClient` level, above the backend
+   seam: a backend only learns that a payload is opaque bytes which it
+   hands over untouched. A minimal `ChannelPublisher` keeps external
+   producers to a few lines. The representative demos (01, 04) run
+   their sensor as a separate process; the timer-driven demos (06, 07)
+   stay as the internal-producer examples. Tested: two twins on one
+   channel both receive every message.
 
 ### M1 — the ORBIT plugin (needs P1; acceptance needs P2)
 
@@ -453,6 +482,12 @@ plan; see status header.)
   RHAPSODY-layer home (after M3 proves the second backend); a
   serializable stream-endpoint descriptor arrives with the first
   out-of-process stream consumer.
+- A channel registry. v1 has no catalogue of external channels and
+  no schema for their payloads: a twin binds a channel by name and
+  trusts the codec its binding names, so producer and consumer agree
+  out of band. Discovery, declared payload schemas and per-channel
+  policy (which codecs a deployment accepts) belong with the
+  declarative twin spec.
 - Per-session subprocess isolation of twin control planes (blast
   radius; §7 R2).
 - **Scale-out: delegated DT hosting** — endpoint-hosted `dt` plugins

From 5a9f6393fb49f2fc5a0a335d1eae40af3f0261c7 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:30:14 +0200
Subject: [PATCH 13/19] Bounded stream connect, embedded broker addresses,
 public get_inference

M0 follow-ups the service depends on:

- `ZMQ_PS_Client.connect(timeout=)` (and `connect_stream_client`) is
  bounded and closes the half-connected client on failure.  The twin's
  background initialization must fail into `failed` + a last error, never
  park in `initializing` because the stream broker is unreachable.
- `config.embedded_stream_addresses()`: an unconfigured *embedded* broker
  takes a random loopback port (it reports what it bound), where the
  standalone demo broker keeps the fixed default ports.
- `DTRuntime.get_inference()`: public entry to the inference path the
  service exposes as a verb, so it stops reaching into a private method.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/config.py    | 15 ++++++++++
 src/digitaltwin/runtime.py   | 15 ++++++++++
 src/digitaltwin/streaming.py | 57 +++++++++++++++++++++++++++---------
 3 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/src/digitaltwin/config.py b/src/digitaltwin/config.py
index c72bfe1..b43292a 100644
--- a/src/digitaltwin/config.py
+++ b/src/digitaltwin/config.py
@@ -41,3 +41,18 @@ def stream_addresses(
         pub_addr or os.environ.get(ENV_PUB_ADDR) or DEFAULT_PUB_ADDR,
         sub_addr or os.environ.get(ENV_SUB_ADDR) or DEFAULT_SUB_ADDR,
     )
+
+
+def embedded_stream_addresses() -> tuple[str, str]:
+    """Resolve the bind addresses of a service-embedded stream broker.
+
+    Same environment variables as `stream_addresses`, but an unconfigured
+    embedded broker takes a random loopback port instead of the fixed
+    demo ports -- it reports what it bound, so nothing has to agree on a
+    number up front.
+    """
+
+    return (
+        os.environ.get(ENV_PUB_ADDR) or RANDOM_PUB_ADDR,
+        os.environ.get(ENV_SUB_ADDR) or RANDOM_SUB_ADDR,
+    )
diff --git a/src/digitaltwin/runtime.py b/src/digitaltwin/runtime.py
index e54a582..c298b53 100644
--- a/src/digitaltwin/runtime.py
+++ b/src/digitaltwin/runtime.py
@@ -570,6 +570,21 @@ def add_agent(
         rt._internal_add_investigator = self._internal_add_investigator
         self._to_asyncio_task(agent.main_loop, rt, *args, **kwargs)
 
+    async def get_inference(
+        self, in_data: TypedData, output_dtype: DataType
+    ) -> TypedData:
+        """Run one inference through the graph and return its result.
+
+        The out-of-graph entry point (the service exposes it as a verb);
+        components use `RuntimeAPI.get_inference`, which is the same path.
+        `NULL_DTYPE` comes back when no component serves the requested
+        input -> output mapping.
+        """
+
+        answer = await self._internal_agent_inference(in_data, output_dtype)
+
+        return answer if answer is not None else TypedData(NULL_DTYPE, None)
+
     # agent-to-agent inference communication
     async def _internal_agent_inference(self, in_data: TypedData, req_dtype: DataType):
         # is there an agent registered that can handle in -> request_out?
diff --git a/src/digitaltwin/streaming.py b/src/digitaltwin/streaming.py
index 7260aa4..71572e2 100644
--- a/src/digitaltwin/streaming.py
+++ b/src/digitaltwin/streaming.py
@@ -69,6 +69,10 @@ def decode_payload(payload: bytes, codec: str):
 BROKER_START_TIMEOUT = 30.0
 BROKER_STOP_TIMEOUT = 5.0
 
+# bounded connect: a client which cannot reach the broker must fail, not
+# wait forever (the service turns that failure into a twin state)
+CLIENT_CONNECT_TIMEOUT = 30.0
+
 
 class PubSubBackend(ABC):
     label = "generic"
@@ -337,15 +341,37 @@ async def _connect_socket(self, sock, addr):
             sock.disable_monitor()
             monitor.close(linger=0)
 
-    async def connect(self):
+    async def connect(self, timeout: Optional[float] = CLIENT_CONNECT_TIMEOUT):
+        """Connect the sockets, bounded by `timeout` (None waits forever).
+
+        On timeout the half-connected client is closed before the error
+        propagates -- an unreachable broker must not leak sockets, and it
+        must not park a caller (the service turns the error into a failed
+        twin instead).
+        """
+
         self._check_open()
 
-        if self.sub_soc is not None:
-            logger.info("Waiting to connect to ZMQ broker...")
-            await self._connect_socket(self.sub_soc, self.sub_addr)
+        try:
+            async with asyncio.timeout(timeout):
+                if self.sub_soc is not None:
+                    logger.info("Waiting to connect to ZMQ broker...")
+                    await self._connect_socket(self.sub_soc, self.sub_addr)
+
+                if self.pub_soc is not None:
+                    await self._connect_socket(self.pub_soc, self.pub_addr)
+
+        except TimeoutError:
+            addrs = f"{self.sub_addr} / {self.pub_addr}"
+            await self.close()
+            raise TimeoutError(
+                f"stream broker at {addrs} did not accept a connection"
+                f" within {timeout}s"
+            ) from None
 
-        if self.pub_soc is not None:
-            await self._connect_socket(self.pub_soc, self.pub_addr)
+        except BaseException:
+            await self.close()
+            raise
 
         if self.sub_soc is not None:
             self._task = asyncio.create_task(self._run())
@@ -703,11 +729,7 @@ async def connect_backend(self, timeout: Optional[float] = None) -> PubSubBacken
             )
 
         backend = ZMQ_PS_Client(self.pub_addr, self.sub_addr)
-        try:
-            await asyncio.wait_for(backend.connect(), timeout)
-        except BaseException:
-            await backend.close()
-            raise
+        await backend.connect(timeout)
 
         return backend
 
@@ -767,8 +789,15 @@ async def close(self):
 
 
 async def connect_stream_client(
-    namespace: str, pub_addr: Optional[str] = None, sub_addr: Optional[str] = None
+    namespace: str,
+    pub_addr: Optional[str] = None,
+    sub_addr: Optional[str] = None,
+    timeout: Optional[float] = CLIENT_CONNECT_TIMEOUT,
 ) -> PubSubClient:
-    """Build and connect a namespaced stream client from configuration."""
+    """Build and connect a namespaced stream client from configuration.
+
+    The connect is bounded by `timeout` -- see `ZMQ_PS_Client.connect`.
+    """
 
-    return await PubSubConfig.resolve(namespace, pub_addr, sub_addr).connect()
+    cfg = PubSubConfig.resolve(namespace, pub_addr, sub_addr)
+    return await cfg.connect(timeout)

From 7aee148791229952311b6e86d3ec2c78bc465fee Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:30:14 +0200
Subject: [PATCH 14/19] The dt ORBIT plugin: sessions, twins, client, wire
 format

`src/digitaltwin/service/` implements the plugin of the DTaaS plan's
section 3, registered out-of-tree via the `radical.orbit.plugins` entry
point (`--plugins default,dt`).

- `PluginDT`: forces `persistent` sessions and relaxes the owner check so
  the sid is a bearer capability (reattach by sid after a disconnect);
  owns the embedded DT stream broker -- one supervised subprocess shared
  plugin-wide, loopback by default, respawned on the addresses it
  reported; `admin/sessions` is the single admin route.
- `DTSession`: n twins plus the session-shared engines they run on (M1
  builds exactly one, `'task'`, an `OrbitExecutionBackend` with
  `batch_window=0` and `backends=['concurrent']` by default).  Teardown
  stops the twins, then bounds `flow.shutdown()` with `wait_for` -- a
  bare await would hang on asyncflow's unbounded gather.
- `TwinInstance`: `DTRuntime` + its own twin-id-namespaced stream client,
  with the `initializing -> ready -> running -> stopped | failed` state
  machine and a last error.
- API model: `twin_create` is the only asynchronous verb (client-supplied
  uuid, background init, returns `initializing`); everything else is a
  short request/response carrying exactly one graph verb.  `start` /
  `stop` / `twin_close` are idempotent no-ops on a twin already in that
  state; `twin_list` is the only observation mechanism.
- `wire`: cloudpickle-base64 payloads with a client-side size check
  against the 4 MiB frame cap and a Python/cloudpickle version stamp the
  service rejects on skew.
- The persistent-component guard warns when a component registered
  `function_task`s at instantiation and is then added as persistent --
  the one place where both facts are visible.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 pyproject.toml                      |  14 +-
 src/digitaltwin/service/__init__.py |  21 ++
 src/digitaltwin/service/client.py   | 289 +++++++++++++++++
 src/digitaltwin/service/plugin.py   | 379 ++++++++++++++++++++++
 src/digitaltwin/service/session.py  | 485 ++++++++++++++++++++++++++++
 src/digitaltwin/service/wire.py     | 126 ++++++++
 6 files changed, 1313 insertions(+), 1 deletion(-)
 create mode 100644 src/digitaltwin/service/__init__.py
 create mode 100644 src/digitaltwin/service/client.py
 create mode 100644 src/digitaltwin/service/plugin.py
 create mode 100644 src/digitaltwin/service/session.py
 create mode 100644 src/digitaltwin/service/wire.py

diff --git a/pyproject.toml b/pyproject.toml
index 8d9ae10..29e6ac7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,15 +15,27 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+# the ORBIT service plugin (`digitaltwin.service`) and its client
+service = [
+    "radical.orbit>=0.3",
+]
 test = [
     "pytest>=8",
     "pytest-asyncio>=0.24",
 ]
 
+# discovered and imported by the ORBIT plugin host at startup; a broker or
+# endpoint then hosts it with `--plugins default,dt`
+[project.entry-points."radical.orbit.plugins"]
+dt = "digitaltwin.service.plugin:PluginDT"
+
 [tool.setuptools.packages.find]
 where = ["src"]
 
 [tool.pytest.ini_options]
-testpaths = ["test/unit"]
+testpaths = ["test/unit", "test/integration"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"
+markers = [
+    "integration: needs a live ORBIT broker and rhapsody endpoint",
+]
diff --git a/src/digitaltwin/service/__init__.py b/src/digitaltwin/service/__init__.py
new file mode 100644
index 0000000..e070c6c
--- /dev/null
+++ b/src/digitaltwin/service/__init__.py
@@ -0,0 +1,21 @@
+"""The `dt` ORBIT plugin: digital twins as a long-running service.
+
+Importing this module registers the plugin (ORBIT auto-registers any
+`Plugin` subclass carrying a `plugin_name`); the `radical.orbit.plugins`
+entry point in `pyproject.toml` is what makes a broker or endpoint find
+it without patching ORBIT.
+"""
+
+from .client import DTClient
+from .plugin import PluginDT
+from .session import DTSession, TwinInstance
+from .wire import Package, register_user_modules
+
+__all__ = [
+    "DTClient",
+    "DTSession",
+    "Package",
+    "PluginDT",
+    "TwinInstance",
+    "register_user_modules",
+]
diff --git a/src/digitaltwin/service/client.py b/src/digitaltwin/service/client.py
new file mode 100644
index 0000000..8fea2d4
--- /dev/null
+++ b/src/digitaltwin/service/client.py
@@ -0,0 +1,289 @@
+"""Client helper for the `dt` ORBIT plugin.
+
+One synchronous class.  The shape a demo sees is the local `DTRuntime`
+shape with a twin id in front::
+
+    rt = EndpointRuntime(); rt.start(wait=True)
+
+    dt   = rt.get_plugin('broker', 'dt', config=ENGINES)
+    twin = dt.create_twin()                       # waits for 'ready'
+
+    sensor = dt.package(MySensor)
+    dt.add_task(twin, sensor, TRUTHY, SENSOR_DTYPE, is_persistent=True)
+    dt.start(twin)
+    ...
+    dt.twin_close(twin)
+
+The session survives the client: reattach later with
+`rt.get_plugin('broker', 'dt', sid=<the sid>)` and the twins are still
+running.
+"""
+
+import logging
+import time
+import uuid
+
+from typing import Any, Optional
+
+from radical.orbit.client import PluginClient
+
+from ..components import DataType, TypedData
+from .wire import (
+    Package,
+    decode,
+    encode_checked,
+    register_user_modules,
+    version_stamp,
+)
+
+log = logging.getLogger("radical.orbit")
+
+# how long `create_twin(wait=True)` polls for `ready` -- the service's
+# background init is bounded well below this
+CREATE_TIMEOUT = 300.0
+POLL_INTERVAL = 0.5
+
+# [P0-interim] large but finite; the transport's own call timeout (600 s
+# by default, `EndpointRuntime(tuning={'call_timeout': …})`) caps it
+INFERENCE_TIMEOUT = 600.0
+
+READY_STATES = ("ready", "running", "stopped")
+
+
+class DTClient(PluginClient):
+    """Application-side API of the digital twin service."""
+
+    # -- sessions -----------------------------------------------------------
+
+    def register_session(
+        self,
+        sid: Optional[str] = None,
+        lifetime: Optional[str] = None,
+        ttl: Optional[float] = None,
+        config: Optional[dict] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Open a session, or reattach to `sid`.
+
+        `config` carries the engine configuration and applies at create
+        time only::
+
+            {"engines": {"task": {"endpoint_name": "ep1",
+                                  "backends": ["concurrent"]}}}
+
+        Sessions are always persistent (the service forces it), so a
+        `lifetime` / `ttl` argument is accepted and ignored.
+        """
+
+        payload = {"sid": sid} if sid else {}
+        if config is not None:
+            payload["config"] = config
+
+        resp = self._request("POST", self._url("register_session"), json=payload)
+        self._raise(resp, "register_session")
+        self._sid = resp.json()["sid"]
+
+    def admin_sessions(self) -> dict:
+        """Every session on this service, with its twins and their errors."""
+
+        resp = self._request("GET", self._url("admin/sessions"))
+        self._raise(resp, "admin_sessions")
+
+        return resp.json()
+
+    # -- artifacts ----------------------------------------------------------
+
+    @staticmethod
+    def package(cls: type, *args: Any, **kwargs: Any) -> Package:
+        """Ship a component *class* plus its constructor arguments.
+
+        The service instantiates it with the session's engine as the
+        leading `flow` argument -- exactly the local constructor
+        convention.  Classes defined outside an installed package need
+        `register_user_modules([...])` first.
+        """
+
+        return Package(cls, args, kwargs)
+
+    register_user_modules = staticmethod(register_user_modules)
+
+    # -- twins --------------------------------------------------------------
+
+    def create_twin(
+        self,
+        twin_id: Optional[str] = None,
+        config: Optional[dict] = None,
+        wait: bool = True,
+        timeout: float = CREATE_TIMEOUT,
+    ) -> str:
+        """Create a twin and return its id.
+
+        The id is a client-side uuid: re-issuing `create_twin` with the
+        same id after a lost response is a no-op, which is what makes the
+        one asynchronous verb safe to retry.
+
+        The call itself returns as soon as the twin is registered
+        (`initializing`); with `wait` the helper polls `twin_list` until
+        the twin is `ready`.
+        """
+
+        twin_id = twin_id or str(uuid.uuid4())
+
+        payload = {"twin_id": twin_id}
+        if config is not None:
+            payload["config"] = config
+
+        resp = self._post(f"twin_create/{self.sid}", payload, "create_twin")
+        if wait and resp["state"] not in READY_STATES:
+            self.wait_ready(twin_id, timeout)
+
+        return twin_id
+
+    def wait_ready(self, twin_id: str, timeout: float = CREATE_TIMEOUT) -> str:
+        """Poll `twin_list` until `twin_id` leaves `initializing`."""
+
+        deadline = time.time() + timeout
+
+        while True:
+            twin = self.twin(twin_id)
+            state = twin["state"]
+
+            if state in READY_STATES:
+                return state
+            if state == "failed":
+                raise RuntimeError(
+                    f"twin {twin_id} failed to initialize: {twin['last_error']}"
+                )
+            if time.time() > deadline:
+                raise TimeoutError(
+                    f"twin {twin_id} still {state} after {timeout}s"
+                )
+
+            time.sleep(POLL_INTERVAL)
+
+    def twin_list(self) -> list[dict]:
+        """State of every twin in this session -- the observation path."""
+
+        self._require_session()
+        resp = self._request("GET", self._url(f"twin_list/{self.sid}"))
+        self._raise(resp, "twin_list")
+
+        return resp.json()["twins"]
+
+    def twin(self, twin_id: str) -> dict:
+        """One twin's entry from `twin_list`."""
+
+        for twin in self.twin_list():
+            if twin["twin_id"] == twin_id:
+                return twin
+
+        raise RuntimeError(f"unknown twin: {twin_id}")
+
+    def twin_close(self, twin_id: str) -> str:
+        """Stop and forget a twin.  A no-op on an already closed one."""
+
+        return self._post(f"twin_close/{self.sid}/{twin_id}", None, "twin_close")[
+            "state"
+        ]
+
+    # -- graph verbs --------------------------------------------------------
+
+    def add_task(
+        self,
+        twin_id: str,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        is_persistent: bool = False,
+    ) -> str:
+        return self._verb(
+            twin_id, "add_task", package, input_dtype, output_dtype, is_persistent
+        )["state"]
+
+    def add_investigator(
+        self,
+        twin_id: str,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        *args: Any,
+        **kwargs: Any,
+    ) -> str:
+        return self._verb(
+            twin_id,
+            "add_investigator",
+            package,
+            input_dtype,
+            output_dtype,
+            *args,
+            **kwargs,
+        )["state"]
+
+    def add_agent(
+        self,
+        twin_id: str,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        *args: Any,
+        **kwargs: Any,
+    ) -> str:
+        return self._verb(
+            twin_id, "add_agent", package, input_dtype, output_dtype, *args, **kwargs
+        )["state"]
+
+    def start(self, twin_id: str) -> str:
+        """Start a twin.  Starting a running twin is a no-op."""
+
+        return self._verb(twin_id, "start")["state"]
+
+    def stop(self, twin_id: str) -> str:
+        """Stop a twin.  Terminal in v1, and a no-op once stopped."""
+
+        return self._verb(twin_id, "stop")["state"]
+
+    def describe(self, twin_id: str) -> dict:
+        """The twin's serializable graph summary."""
+
+        return self._verb(twin_id, "describe")["graph"]
+
+    def get_inference(
+        self,
+        twin_id: str,
+        in_data: TypedData,
+        output_dtype: DataType,
+        timeout: float = INFERENCE_TIMEOUT,
+    ) -> TypedData:
+        """Run one inference through the twin and return its result."""
+
+        answer = self._verb(
+            twin_id, "get_inference", in_data, output_dtype, timeout=timeout
+        )
+
+        return decode(answer["inference"])
+
+    # -- transport ----------------------------------------------------------
+
+    def _verb(self, twin_id: str, verb: str, *args: Any, **kwargs: Any) -> dict:
+        """One graph verb, one request.
+
+        Arguments are cloudpickled as a whole (dtypes, packages and
+        inference payloads are all arbitrary Python) and size-checked
+        against ORBIT's frame cap before they hit the wire.
+        """
+
+        payload = {
+            "verb": verb,
+            "payload": encode_checked({"args": args, "kwargs": kwargs},
+                                      f"{verb} payload"),
+            "client": version_stamp(),
+        }
+
+        return self._post(f"twin_call/{self.sid}/{twin_id}", payload, verb)
+
+    def _post(self, path: str, payload: Optional[dict], context: str) -> dict:
+        self._require_session()
+        resp = self._request("POST", self._url(path), json=payload)
+        self._raise(resp, context)
+
+        return resp.json()
diff --git a/src/digitaltwin/service/plugin.py b/src/digitaltwin/service/plugin.py
new file mode 100644
index 0000000..9e96930
--- /dev/null
+++ b/src/digitaltwin/service/plugin.py
@@ -0,0 +1,379 @@
+"""The `dt` ORBIT plugin: digital twins as a long-running service.
+
+Hosted on a standalone broker by default (`--plugins default,dt`);
+endpoint hosting works the same way.  The plugin owns
+
+- the session policy: sessions are forced `persistent` (twins survive
+  disappearing clients, without timeout) and the sid acts as a bearer
+  capability, so a reconnecting client reattaches by sid alone;
+- the embedded DT stream broker: one supervised subprocess shared by
+  every session, respawned on the addresses it reported;
+- the wire: one graph verb per `twin_call`, cloudpickle-base64 payloads
+  with a version stamp, and `twin_list` as the only observation path.
+
+**Binding policy (risk R7)**: the stream broker binds loopback by
+default.  Its payloads are cloudpickled, so anyone who can reach its
+ports executes code in every subscriber.  A non-loopback bind needs an
+explicit `DT_STREAM_PUB_ADDR` / `DT_STREAM_SUB_ADDR` configuration *and*
+a firewalled/private network.
+"""
+
+import asyncio
+import logging
+import os
+import time
+
+from typing import Any, Optional
+
+from fastapi import FastAPI
+from radical.orbit.errors import http_exception
+from radical.orbit.plugin_base import Plugin
+from starlette.requests import Request
+
+from ..config import embedded_stream_addresses
+from ..streaming import ZMQ_BrokerProcess
+from .client import DTClient
+from .session import VERBS, DTSession
+from .wire import check_versions, decode
+
+log = logging.getLogger("radical.orbit")
+
+# route templates -- single-sourced between registration and the client
+ROUTE_TWIN_CREATE = "twin_create/{sid}"
+ROUTE_TWIN_LIST = "twin_list/{sid}"
+ROUTE_TWIN_CLOSE = "twin_close/{sid}/{twin_id}"
+ROUTE_TWIN_CALL = "twin_call/{sid}/{twin_id}"
+ROUTE_ADMIN_SESSIONS = "admin/sessions"
+
+# how often the supervisor checks that the stream broker is still alive
+BROKER_WATCH_INTERVAL = 5.0
+
+# plugin-level ORBIT broker URL for the engines' OrbitExecutionBackend;
+# unset lets ORBIT resolve it (env / ~/.radical/orbit), as does the token
+ENV_BROKER_URL = "DT_ORBIT_BROKER_URL"
+
+
+class PluginDT(Plugin):
+    """Digital-twin plugin for ORBIT.
+
+    - POST `/dt/register_session`            -- open (or reattach to) a session
+    - POST `/dt/twin_create/{sid}`           -- register a twin, init in background
+    - GET  `/dt/twin_list/{sid}`             -- states of all twins in a session
+    - POST `/dt/twin_close/{sid}/{twin_id}`  -- stop and forget one twin
+    - POST `/dt/twin_call/{sid}/{twin_id}`   -- exactly one graph verb
+    - GET  `/dt/admin/sessions`              -- every session, twin and error
+
+    Every call is short except `get_inference`.  No notifications, no
+    request ids: `twin_list` polling is the observation mechanism.
+    """
+
+    plugin_name = "dt"
+    session_class = DTSession
+    client_class = DTClient
+    version = "0.1.0"
+
+    ui_config = {
+        "icon": "🪞",
+        "title": "Digital Twins",
+        "description": "Host long-running digital twins (in-situ inference).",
+    }
+
+    def __init__(self, app: FastAPI, instance_name: str = "dt"):
+        super().__init__(app, instance_name)
+
+        self.broker_url: Optional[str] = os.environ.get(ENV_BROKER_URL) or None
+
+        # the embedded DT stream broker, shared plugin-wide and started on
+        # first need (see `stream_addresses`)
+        self._stream_broker: Optional[ZMQ_BrokerProcess] = None
+        self._stream_addrs: Optional[tuple[str, str]] = None
+        self._stream_lock = asyncio.Lock()
+        self._supervisor: Optional[asyncio.Task] = None
+
+        self.add_route_post(ROUTE_TWIN_CREATE, self.twin_create)
+        self.add_route_get(ROUTE_TWIN_LIST, self.twin_list)
+        self.add_route_post(ROUTE_TWIN_CLOSE, self.twin_close)
+        self.add_route_post(ROUTE_TWIN_CALL, self.twin_call)
+        self.add_route_get(ROUTE_ADMIN_SESSIONS, self.admin_sessions)
+
+    # -- session policy -----------------------------------------------------
+
+    def _normalize_session_policy(self, data: dict) -> tuple:
+        """Force `persistent`.
+
+        Twins run for days and clients attach opportunistically; a session
+        that expires under an absent client would take its twins with it.
+        Teardown is explicit only (`unregister_session`).
+        """
+
+        forced = dict(data or {})
+        forced["lifetime"] = "persistent"
+        forced.pop("ttl", None)
+
+        return super()._normalize_session_policy(forced)
+
+    def _check_owner(self, sid: str, owner: Optional[str]) -> None:
+        """The sid is a bearer capability: any holder may reattach.
+
+        A client that reconnects comes back as a *different* participant,
+        so the owner check would lock it out of its own twins.  This is
+        acceptable inside ORBIT's single-token trust domain, and the
+        recorded owner stays visible in `admin/sessions`.
+        """
+
+    async def register_session(self, request: Request) -> dict:
+        """Open or reattach to a session.
+
+        Body (all optional): `{"sid": str, "config": {...}}`.  `config`
+        carries the engine configuration and applies at create time only:
+
+            {"engines": {"task": {"endpoint_name": "ep1",
+                                  "backends": ["concurrent"]}}}
+        """
+
+        self._ensure_cleanup_task()
+
+        data = await _body(request)
+        config = data.get("config")
+        if config is not None and not isinstance(config, dict):
+            raise http_exception(ValueError("'config' must be an object"))
+
+        owner = self._request_owner(request)
+        sid, lifetime, ttl = self._normalize_session_policy(data)
+
+        if sid is not None and sid in self._sessions:
+            self._check_owner(sid, owner)
+            self._check_policy_conflict(sid, lifetime, ttl)
+            self._touch(sid)
+            log.info("[dt] reattached session %s", sid)
+            return {"sid": sid, "reattached": True}
+
+        sid = await self._open_session(sid, lifetime, ttl, owner)
+        session = self._sessions[sid]
+        session.config = config or {}
+
+        return {"sid": sid, "reattached": False}
+
+    # -- routes -------------------------------------------------------------
+
+    async def twin_create(self, request: Request) -> dict:
+        """Register a twin; engine and stream come up in the background.
+
+        Body: `{"twin_id": <client-supplied uuid>, "config": {...}}`.
+        The client-supplied id is what makes a retry idempotent, and it
+        doubles as the twin's stream namespace -- hence globally unique
+        across sessions, not just within one.
+        """
+
+        sid = request.path_params["sid"]
+        data = await _body(request)
+
+        twin_id = _twin_id(data.get("twin_id"))
+        owner = self._twin_owner(twin_id)
+        if owner not in (None, sid):
+            raise http_exception(
+                ValueError(f"twin id {twin_id} is already used by session {owner}")
+            )
+
+        config = data.get("config")
+        if config is not None and not isinstance(config, dict):
+            raise http_exception(ValueError("'config' must be an object"))
+
+        return await self._forward(
+            sid, DTSession.twin_create, twin_id=twin_id, config=config
+        )
+
+    async def twin_list(self, request: Request) -> dict:
+        return await self._forward(request.path_params["sid"], DTSession.twin_list)
+
+    async def twin_close(self, request: Request) -> dict:
+        return await self._forward(
+            request.path_params["sid"],
+            DTSession.twin_close,
+            twin_id=request.path_params["twin_id"],
+        )
+
+    async def twin_call(self, request: Request) -> dict:
+        """Apply one graph verb.
+
+        Body: `{"verb": str, "payload": <b64 cloudpickle of
+        {"args": [...], "kwargs": {...}}>, "client": {"python":…,
+        "cloudpickle":…}}`.
+        """
+
+        sid = request.path_params["sid"]
+        twin_id = request.path_params["twin_id"]
+        data = await _body(request)
+
+        verb = data.get("verb")
+        if verb not in VERBS:
+            raise http_exception(
+                ValueError(f"unknown verb {verb!r}; expected one of {', '.join(VERBS)}")
+            )
+
+        try:
+            check_versions(data.get("client"))
+            call = decode(data.get("payload") or "") if data.get("payload") else {}
+        except ValueError as exc:
+            raise http_exception(exc) from exc
+        except Exception as exc:
+            raise http_exception(
+                ValueError(f"undecodable {verb} payload: {exc}")
+            ) from exc
+
+        return await self._forward(
+            sid,
+            DTSession.twin_call,
+            twin_id=twin_id,
+            verb=verb,
+            args=tuple(call.get("args") or ()),
+            kwargs=dict(call.get("kwargs") or {}),
+        )
+
+    async def admin_sessions(self, request: Request) -> dict:
+        """Every session with its owner, age, twins, states and errors.
+
+        The single admin route: immortal sessions leak by design, so they
+        have to be discoverable.  Teardown then uses the ordinary
+        `twin_close` / `unregister_session` routes with a sid from here.
+        """
+
+        now = time.time()
+        sessions = []
+
+        for sid, session in self._sessions.items():
+            record = self._records.get(sid)
+            entry = {
+                "owner": record.owner if record else None,
+                "lifetime": record.lifetime if record else None,
+                "idle": round(now - record.last_access, 3) if record else None,
+            }
+            if isinstance(session, DTSession):
+                entry.update(session.summary())
+            else:
+                entry["sid"] = sid
+            sessions.append(entry)
+
+        return {
+            "sessions": sessions,
+            "stream_broker": {
+                "addresses": self._stream_addrs,
+                "alive": bool(self._stream_broker and self._stream_broker.is_alive()),
+            },
+        }
+
+    # -- embedded stream broker --------------------------------------------
+
+    async def stream_addresses(self) -> tuple[str, str]:
+        """The DT stream broker's `(publish, subscribe)` addresses.
+
+        Starts the broker on first need and arms the supervisor.  One
+        broker per plugin, shared by every session and twin -- topic
+        namespacing (the twin id) keeps them apart.
+        """
+
+        async with self._stream_lock:
+            if self._stream_broker is None:
+                self._stream_broker = ZMQ_BrokerProcess(*embedded_stream_addresses())
+                self._stream_addrs = await self._stream_broker.start()
+                log.info("[dt] stream broker at %s / %s", *self._stream_addrs)
+
+            if self._supervisor is None:
+                self._supervisor = asyncio.create_task(self._supervise())
+
+            return self._stream_addrs
+
+    async def _supervise(self) -> None:
+        """Respawn the stream broker if it dies.
+
+        On the *reported* addresses, not the configured ones: those are
+        wildcards by default, and clients have already connected to the
+        concrete ports.  ZMQ reconnects them on its own, so a respawn is
+        invisible above this line -- whereas a silently dead broker would
+        stall every stream on a days-long twin.
+        """
+
+        while True:
+            try:
+                await asyncio.sleep(BROKER_WATCH_INTERVAL)
+
+                async with self._stream_lock:
+                    if self._stream_broker is None or self._stream_broker.is_alive():
+                        continue
+
+                    log.warning("[dt] stream broker died -- respawning on %s / %s",
+                                *self._stream_addrs)
+                    await self._stream_broker.stop()
+                    self._stream_broker = ZMQ_BrokerProcess(*self._stream_addrs)
+                    self._stream_addrs = await self._stream_broker.start()
+
+            except asyncio.CancelledError:
+                return
+            except Exception:
+                log.exception("[dt] stream broker supervision failed")
+
+    # -- teardown -----------------------------------------------------------
+
+    async def shutdown(self) -> None:
+        """Host-shutdown path: supervisor, then sessions, then the broker."""
+
+        supervisor, self._supervisor = self._supervisor, None
+        if supervisor is not None:
+            supervisor.cancel()
+            try:
+                await supervisor
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                log.exception("[dt] stream broker supervisor teardown")
+
+        await super().shutdown()
+
+        broker, self._stream_broker = self._stream_broker, None
+        if broker is not None:
+            await broker.stop()
+        self._stream_addrs = None
+
+    # -- internals ----------------------------------------------------------
+
+    def _twin_owner(self, twin_id: str) -> Optional[str]:
+        """The sid currently holding `twin_id`, if any.
+
+        Twin ids are globally unique because the stream broker is shared
+        plugin-wide; derived by scan so there is no registry to keep in
+        sync with session and twin teardown.
+        """
+
+        for sid, session in self._sessions.items():
+            if isinstance(session, DTSession) and twin_id in session.twins:
+                return sid
+        return None
+
+
+async def _body(request: Request) -> dict:
+    """Best-effort JSON body as a dict (missing / malformed -> empty)."""
+
+    try:
+        data = await request.json()
+    except Exception:
+        return {}
+
+    return data if isinstance(data, dict) else {}
+
+
+def _twin_id(twin_id: Any) -> str:
+    """Validate a client-supplied twin id.
+
+    It becomes the twin's stream namespace, where a separator would let
+    two twins alias each other's topics.
+    """
+
+    if not isinstance(twin_id, str) or not twin_id or len(twin_id) > 128:
+        raise http_exception(ValueError("'twin_id' must be a non-empty string"))
+
+    if any(c in twin_id for c in "/\x00 "):
+        raise http_exception(
+            ValueError(f"invalid twin id {twin_id!r}: no '/', spaces or NULs")
+        )
+
+    return twin_id
diff --git a/src/digitaltwin/service/session.py b/src/digitaltwin/service/session.py
new file mode 100644
index 0000000..d07a2a0
--- /dev/null
+++ b/src/digitaltwin/service/session.py
@@ -0,0 +1,485 @@
+"""Service-side session and twin instances.
+
+A `DTSession` belongs to one client and hosts many independent twins.
+It owns the session-shared execution engines (M1: exactly one, `'task'`);
+each `TwinInstance` owns a `DTRuntime` plus its own namespaced stream
+client.  Twin teardown never disturbs its siblings or the engines.
+"""
+
+import asyncio
+import contextlib
+import logging
+import time
+
+from typing import Any, Optional
+
+from fastapi import HTTPException
+from radical.asyncflow import WorkflowEngine  # type: ignore
+from radical.orbit.plugin_session_base import PluginSession
+from rhapsody.backends.execution.orbit import OrbitExecutionBackend  # type: ignore
+
+from ..components import DataType, TypedData
+from ..runtime import DTRuntime
+from ..streaming import PubSubClient, connect_stream_client
+from .wire import Package, encode
+
+log = logging.getLogger("radical.orbit")
+
+# the one engine M1 knows about; M2 adds 'exsitu' as a config addition
+TASK_ENGINE = "task"
+
+# co-located-demo default -- 'dragon_v3' (the rhapsody default) would
+# break every demo on a laptop
+DEFAULT_BACKENDS = ["concurrent"]
+
+# bounded waits.  This runs for days on a shared event loop: nothing here
+# may block it and nothing may hang.
+TWIN_INIT_TIMEOUT = 300.0  # engine (<=150s) + stream connect + slack
+STREAM_CONNECT_TIMEOUT = 30.0
+TWIN_STOP_TIMEOUT = 10.0
+ENGINE_SHUTDOWN_TIMEOUT = 30.0
+
+# get_inference is the only potentially-long verb; the client may pick its
+# own bound.  [P0-interim] large but finite -- see the plan's section 3.
+DEFAULT_INFERENCE_TIMEOUT = 600.0
+
+STATE_INITIALIZING = "initializing"
+STATE_FAILED = "failed"
+STATE_CLOSED = "closed"
+
+# exactly one of these per twin_call
+VERBS = (
+    "add_task",
+    "add_investigator",
+    "add_agent",
+    "start",
+    "stop",
+    "describe",
+    "get_inference",
+)
+
+
+class TwinInstance:
+    """One twin: a `DTRuntime`, its stream client, and a state machine.
+
+    `initializing -> ready -> running -> stopped | failed`.  The first
+    state is owned here (the twin has no runtime yet); from `ready` on,
+    the runtime's own state machine is the truth.
+    """
+
+    def __init__(self, twin_id: str, config: Optional[dict] = None):
+        self.twin_id = twin_id
+        self.config = config or {}
+        self.created = time.time()
+
+        self.runtime: Optional[DTRuntime] = None
+        self.stream: Optional[PubSubClient] = None
+
+        self._state = STATE_INITIALIZING
+        self._last_error: Optional[str] = None
+
+        self._init_task: Optional[asyncio.Task] = None
+        # in-flight get_inference calls: they are awaited by request
+        # handlers, not by the runtime, so teardown has to cancel them
+        # explicitly or a closing twin would leave a caller hanging
+        self._inflight: set[asyncio.Task] = set()
+
+    @property
+    def state(self) -> str:
+        return self._state if self.runtime is None else str(self.runtime.state)
+
+    @property
+    def last_error(self) -> Optional[str]:
+        if self.runtime is not None and self.runtime.last_error:
+            return self.runtime.last_error
+        return self._last_error
+
+    def summary(self) -> dict:
+        """The twin's entry in `twin_list` / `admin/sessions`."""
+
+        return {
+            "twin_id": self.twin_id,
+            "state": self.state,
+            "last_error": self.last_error,
+            "age": round(time.time() - self.created, 3),
+            "config": self.config,
+        }
+
+    def ready(self, runtime: DTRuntime, stream: PubSubClient) -> None:
+        self.runtime = runtime
+        self.stream = stream
+
+    def fail(self, error: BaseException | str) -> None:
+        self._state = STATE_FAILED
+        self._last_error = (
+            error if isinstance(error, str) else f"{type(error).__name__}: {error}"
+        )
+        log.error("[dt] twin %s failed: %s", self.twin_id, self._last_error)
+
+    def track(self, task: asyncio.Task) -> asyncio.Task:
+        """Register an in-flight request-side task for teardown."""
+
+        self._inflight.add(task)
+        task.add_done_callback(self._inflight.discard)
+
+        return task
+
+    async def close(self, timeout: float = TWIN_STOP_TIMEOUT) -> None:
+        """Tear the twin down.  Best-effort, bounded, never raises."""
+
+        pending = [t for t in (self._init_task, *self._inflight) if t is not None]
+        for task in pending:
+            task.cancel()
+
+        if pending:
+            with contextlib.suppress(Exception):
+                await asyncio.wait(pending, timeout=timeout)
+
+        try:
+            if self.runtime is not None:
+                # stop() also closes the twin's stream client
+                await self.runtime.stop(timeout)
+            elif self.stream is not None:
+                await asyncio.wait_for(self.stream.close(), timeout)
+
+        except Exception as exc:
+            log.warning("[dt] twin %s teardown: %s", self.twin_id, exc)
+
+        self._state = STATE_CLOSED
+
+
+class DTSession(PluginSession):
+    """Per-client session: n twins plus the engines they share."""
+
+    def __init__(self, sid: str, config: Optional[dict] = None):
+        super().__init__(sid)
+
+        self.config = config or {}
+        self.created = time.time()
+
+        self.twins: dict[str, TwinInstance] = {}
+        self._engines: dict[str, WorkflowEngine] = {}
+        self._engine_lock = asyncio.Lock()
+
+    # -- twin lifecycle -----------------------------------------------------
+
+    async def twin_create(self, twin_id: str, config: Optional[dict] = None) -> dict:
+        """Register a twin and kick its initialization in the background.
+
+        Returns immediately -- engine and stream setup can take minutes and
+        must never be paid inside a held request.  Re-creating a live twin
+        id is a no-op reporting the current state, which is what makes a
+        retried `twin_create` safe.
+        """
+
+        self._check_active()
+
+        twin = self.twins.get(twin_id)
+        if twin is None:
+            twin = TwinInstance(twin_id, config)
+            self.twins[twin_id] = twin
+            twin._init_task = asyncio.create_task(self._init_twin(twin))
+            log.info("[dt] session %s created twin %s", self.sid, twin_id)
+
+        return self._twin_state(twin)
+
+    async def twin_list(self) -> dict:
+        """The only observation mechanism in v1: every twin, every state."""
+
+        self._check_active()
+
+        return {"sid": self.sid, "twins": [t.summary() for t in self.twins.values()]}
+
+    async def twin_close(self, twin_id: str) -> dict:
+        """Stop and forget a twin.  Idempotent: closing an unknown (already
+        closed) twin reports `closed` rather than failing a retry."""
+
+        self._check_active()
+
+        twin = self.twins.pop(twin_id, None)
+        if twin is None:
+            return {"twin_id": twin_id, "state": STATE_CLOSED}
+
+        await twin.close()
+        log.info("[dt] session %s closed twin %s", self.sid, twin_id)
+
+        return self._twin_state(twin)
+
+    async def twin_call(
+        self, twin_id: str, verb: str, args: tuple = (), kwargs: Optional[dict] = None
+    ) -> dict:
+        """Apply exactly one graph verb to one twin."""
+
+        self._check_active()
+
+        twin = self._live_twin(twin_id)
+        handler = getattr(self, f"_verb_{verb}")
+
+        try:
+            extra = await handler(twin, *args, **(kwargs or {}))
+
+        except HTTPException:
+            raise
+        except asyncio.CancelledError:
+            # the twin was closed under an in-flight call
+            raise HTTPException(
+                status_code=409, detail=f"twin {twin_id} was closed during {verb}"
+            ) from None
+        except TimeoutError:
+            raise HTTPException(
+                status_code=504, detail=f"twin {twin_id}: {verb} timed out"
+            ) from None
+        except (RuntimeError, ValueError, AssertionError) as exc:
+            # the runtime's own refusals (stopped graph, start-after-stop,
+            # bad dtypes) are client errors, not service faults
+            raise HTTPException(
+                status_code=409, detail=f"twin {twin_id}: {verb}: {exc}"
+            ) from exc
+
+        return {**self._twin_state(twin), **(extra or {})}
+
+    # -- verbs --------------------------------------------------------------
+
+    async def _verb_add_task(
+        self,
+        twin: TwinInstance,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        is_persistent: bool = False,
+    ) -> None:
+        component = self._instantiate(package, twin, is_persistent)
+        twin.runtime.add_task(component, input_dtype, output_dtype, is_persistent)
+
+    async def _verb_add_investigator(
+        self,
+        twin: TwinInstance,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        *args,
+        **kwargs,
+    ) -> None:
+        component = self._instantiate(package, twin)
+        twin.runtime.add_investigator(
+            component, input_dtype, output_dtype, *args, **kwargs
+        )
+
+    async def _verb_add_agent(
+        self,
+        twin: TwinInstance,
+        package: Package,
+        input_dtype: DataType,
+        output_dtype: DataType,
+        *args,
+        **kwargs,
+    ) -> None:
+        component = self._instantiate(package, twin)
+        twin.runtime.add_agent(component, input_dtype, output_dtype, *args, **kwargs)
+
+    async def _verb_start(self, twin: TwinInstance) -> None:
+        # a running twin is left running: an idempotent retry, not an error
+        twin.runtime.start()
+
+    async def _verb_stop(self, twin: TwinInstance) -> None:
+        # terminal and idempotent in the runtime itself
+        await twin.runtime.stop()
+
+    async def _verb_describe(self, twin: TwinInstance) -> dict:
+        return {"graph": twin.runtime.describe()}
+
+    async def _verb_get_inference(
+        self,
+        twin: TwinInstance,
+        in_data: TypedData,
+        output_dtype: DataType,
+        timeout: Optional[float] = DEFAULT_INFERENCE_TIMEOUT,
+    ) -> dict:
+        # run it as a tracked task so `twin_close` can cancel it -- an
+        # in-flight inference must not outlive (or hold up) its twin
+        task = twin.track(
+            asyncio.create_task(twin.runtime.get_inference(in_data, output_dtype))
+        )
+
+        return {"inference": encode(await asyncio.wait_for(task, timeout))}
+
+    # -- engines ------------------------------------------------------------
+
+    async def engine(self, name: str = TASK_ENGINE) -> WorkflowEngine:
+        """The session-shared engine `name`, created on first use.
+
+        One engine per name per session, never per twin.  M1 only ever
+        asks for `'task'`.
+        """
+
+        async with self._engine_lock:
+            flow = self._engines.get(name)
+            if flow is None:
+                flow = self._engines[name] = await self._create_engine(name)
+            return flow
+
+    async def _create_engine(self, name: str) -> WorkflowEngine:
+        cfg = (self.config.get("engines") or {}).get(name) or {}
+
+        log.info(
+            "[dt] session %s building engine %r on endpoint %s",
+            self.sid,
+            name,
+            cfg.get("endpoint_name") or "<auto>",
+        )
+
+        backend = await OrbitExecutionBackend(
+            broker_url=self.broker_url,
+            endpoint_name=cfg.get("endpoint_name"),
+            backends=cfg.get("backends") or DEFAULT_BACKENDS,
+            batch_window=0,  # per-call latency beats batching for in-situ
+        )
+
+        return await WorkflowEngine.create(backend=backend)
+
+    @property
+    def broker_url(self) -> Optional[str]:
+        """Plugin-level broker URL (`None` lets ORBIT resolve it)."""
+
+        return getattr(self._plugin, "broker_url", None)
+
+    # -- teardown -----------------------------------------------------------
+
+    async def close(self) -> dict:
+        """Stop every twin, then shut the engines down.
+
+        Engine shutdown is `wait_for`-bounded: asyncflow's own shutdown is
+        an unbounded gather, so a bare await could park the host loop.
+        """
+
+        for twin in list(self.twins.values()):
+            await twin.close()
+        self.twins.clear()
+
+        for name, flow in self._engines.items():
+            try:
+                await asyncio.wait_for(flow.shutdown(), ENGINE_SHUTDOWN_TIMEOUT)
+            except Exception as exc:
+                log.warning("[dt] session %s: engine %r shutdown: %s",
+                            self.sid, name, exc)
+        self._engines.clear()
+
+        return await super().close()
+
+    def summary(self) -> dict:
+        """This session's entry in the `admin/sessions` listing."""
+
+        return {
+            "sid": self.sid,
+            "active": self.is_active,
+            "age": round(time.time() - self.created, 3),
+            "engines": sorted(self._engines),
+            "twins": [twin.summary() for twin in self.twins.values()],
+        }
+
+    # -- internals ----------------------------------------------------------
+
+    async def _init_twin(self, twin: TwinInstance) -> None:
+        """Background phase of `twin_create`: engine, stream, runtime.
+
+        Every failure lands in `failed` + a last error -- a twin must
+        never be left sitting in `initializing`.
+        """
+
+        try:
+            if self._plugin is None:
+                raise RuntimeError("session is not attached to a dt plugin")
+
+            async with asyncio.timeout(TWIN_INIT_TIMEOUT):
+                flow = await self.engine(TASK_ENGINE)
+                pub_addr, sub_addr = await self._plugin.stream_addresses()
+                stream = await connect_stream_client(
+                    twin.twin_id, pub_addr, sub_addr, STREAM_CONNECT_TIMEOUT
+                )
+                twin.ready(DTRuntime(flow, stream), stream)
+
+            log.info("[dt] twin %s ready", twin.twin_id)
+
+        except asyncio.CancelledError:
+            twin.fail("twin initialization was cancelled")
+            raise
+        except TimeoutError:
+            twin.fail(f"twin initialization exceeded {TWIN_INIT_TIMEOUT}s")
+        except BaseException as exc:
+            twin.fail(exc)
+
+    def _live_twin(self, twin_id: str) -> TwinInstance:
+        """Resolve a twin that can take a verb, or fail fast.
+
+        Graph verbs never wait for an initializing twin -- the client
+        polls `twin_list` for that.
+        """
+
+        twin = self.twins.get(twin_id)
+        if twin is None:
+            raise HTTPException(status_code=404, detail=f"unknown twin: {twin_id}")
+
+        if twin.runtime is None:
+            raise HTTPException(
+                status_code=409,
+                detail=f"twin {twin_id} is {twin.state}"
+                + (f": {twin.last_error}" if twin.last_error else ""),
+            )
+
+        return twin
+
+    def _instantiate(
+        self, package: Any, twin: TwinInstance, is_persistent: bool = False
+    ) -> Any:
+        """Build a component from a shipped class, injecting the engine.
+
+        Also the home of the persistent-component guard: a persistent
+        `main_loop` runs inline on the host loop, so any `function_task`
+        it registered would be cloudpickled to the endpoint and occupy a
+        backend slot for the twin's lifetime.  Instantiation and
+        `is_persistent` are only visible together here, which is why the
+        check lives service-side.
+        """
+
+        if not isinstance(package, Package):
+            raise ValueError(
+                "component argument must be a DTClient.package() result,"
+                f" got {type(package).__name__}"
+            )
+
+        flow = twin.runtime.flow
+        registered = 0
+        original = flow.function_task
+
+        def counting(*args, **kwargs):
+            nonlocal registered
+            registered += 1
+            return original(*args, **kwargs)
+
+        flow.function_task = counting
+        try:
+            component = package.instantiate(flow)
+        finally:
+            flow.function_task = original
+
+        if is_persistent and registered:
+            log.warning(
+                "[dt] twin %s: persistent component %s registered %d"
+                " function_task(s) -- persistent main_loops run inline on"
+                " the service loop, so those tasks would occupy backend"
+                " slots for the twin's lifetime.  Use plain async code and"
+                " runtime.stream instead.",
+                twin.twin_id,
+                type(component).__name__,
+                registered,
+            )
+
+        return component
+
+    @staticmethod
+    def _twin_state(twin: TwinInstance) -> dict:
+        return {
+            "twin_id": twin.twin_id,
+            "state": twin.state,
+            "last_error": twin.last_error,
+        }
diff --git a/src/digitaltwin/service/wire.py b/src/digitaltwin/service/wire.py
new file mode 100644
index 0000000..440b284
--- /dev/null
+++ b/src/digitaltwin/service/wire.py
@@ -0,0 +1,126 @@
+"""Wire format shared by the DT service and its client.
+
+Everything a DT verb carries -- dtypes, component classes, constructor
+arguments, inference payloads -- is arbitrary Python, so it travels as
+base64-encoded cloudpickle inside the JSON envelope.
+
+Two guards ride along, both cheap and both mandatory:
+
+- a **size check** against ORBIT's 4 MiB frame cap, applied client-side
+  where the offending object is still in scope and the error can name it;
+- a **version stamp** (client Python + cloudpickle), rejected server-side
+  on skew -- a pickle from a different interpreter fails in confusing
+  ways deep inside the plugin otherwise.
+"""
+
+import base64
+import sys
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import cloudpickle
+
+try:
+    from radical.orbit.protocol import FRAME_CAP
+except ImportError:  # the client may be installed without ORBIT
+    FRAME_CAP = 4 * 1024 * 1024
+
+# Room for the JSON envelope (verb, ids, keys) around the payload.  The
+# cap applies to the whole packed frame, so the payload must stay below.
+ENVELOPE_MARGIN = 64 * 1024
+MAX_PAYLOAD = FRAME_CAP - ENVELOPE_MARGIN
+
+
+@dataclass
+class Package:
+    """A component class plus its constructor arguments, shipped to the
+    service, which instantiates it injecting the session's engine as the
+    leading `flow` argument.
+
+    Built by `DTClient.package()`; never instantiated by hand.
+    """
+
+    cls: type
+    args: tuple = ()
+    kwargs: dict = field(default_factory=dict)
+
+    def instantiate(self, flow) -> Any:
+        return self.cls(flow, *self.args, **self.kwargs)
+
+
+def register_user_modules(modules: list) -> None:
+    """Ship these modules by value rather than by reference.
+
+    User code living in a demo directory does not exist on the service,
+    so its modules have to travel with the pickle.
+    """
+
+    for module in modules:
+        cloudpickle.register_pickle_by_value(module)
+
+
+def version_stamp() -> dict:
+    """The interpreter/cloudpickle identity a pickle was produced with."""
+
+    return {
+        "python": "%d.%d" % sys.version_info[:2],
+        "cloudpickle": cloudpickle.__version__,
+    }
+
+
+def check_versions(stamp: dict) -> None:
+    """Reject a payload produced by a skewed client.
+
+    Raises:
+        ValueError: on a Python minor-version or cloudpickle
+            minor-version mismatch, naming both sides.
+    """
+
+    ours = version_stamp()
+    theirs = stamp or {}
+
+    for key, mine in ours.items():
+        yours = theirs.get(key)
+        if yours is None:
+            raise ValueError(f"client did not report its {key} version")
+
+        # compare on major.minor: that is the granularity at which
+        # pickle compatibility actually breaks
+        if _minor(yours) != _minor(mine):
+            raise ValueError(
+                f"{key} version skew: client {yours}, service {mine}"
+                f" -- cloudpickled payloads are not portable across those"
+            )
+
+
+def _minor(version: str) -> tuple:
+    return tuple(str(version).split(".")[:2])
+
+
+def encode(obj: Any) -> str:
+    """Cloudpickle `obj` into a JSON-safe string."""
+
+    return base64.b64encode(cloudpickle.dumps(obj)).decode("ascii")
+
+
+def decode(blob: str) -> Any:
+    """Inverse of `encode`."""
+
+    return cloudpickle.loads(base64.b64decode(blob))
+
+
+def encode_checked(obj: Any, what: str) -> str:
+    """`encode`, refusing payloads that cannot survive the frame cap."""
+
+    blob = encode(obj)
+
+    if len(blob) > MAX_PAYLOAD:
+        raise ValueError(
+            f"{what} is {len(blob) / 1024**2:.1f} MiB encoded, over the"
+            f" {MAX_PAYLOAD / 1024**2:.1f} MiB the ORBIT frame cap leaves"
+            f" for it -- keep bulk data out of the control plane (stage it"
+            f" or stream it)"
+        )
+
+    return blob

From 116f9fb411f1acf8a28181d7cc5318cdae8f5da6 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:45:45 +0200
Subject: [PATCH 15/19] Port the remote demo to the dt plugin, retire the ZMQ
 prototype

`test/09-remote` becomes `test/09-service` and drives the twin through
`DTClient` over ORBIT instead of the hand-rolled ZMQ REQ/REP prototype,
which is deleted along with its host script
(`src/digitaltwin/remote/`, `test/remote_service.py`).

`model.py` also splits its inference in two: the engine task returns a
plain value and the component wraps it in `TypedData`.  ORBIT's rhapsody
plugin cloudpickles task *arguments* but JSON-encodes return values,
stringifying anything that is neither JSON-safe nor bytes
(`plugin_rhapsody.py:547-556`) -- so a DT-typed object cannot come back
out of a task.  Noted in the demo and its README.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .gitignore                                  |   4 +
 src/digitaltwin/remote/README.md            |  99 --------------
 src/digitaltwin/remote/client.py            | 136 --------------------
 src/digitaltwin/remote/remote_service.py    | 117 -----------------
 test/09-remote/README.md                    |  25 ----
 test/09-remote/model.py                     |  39 ------
 test/09-remote/run_me.py                    |  62 ---------
 test/09-service/README.md                   |  52 ++++++++
 test/{09-remote => 09-service}/agent.py     |   0
 test/{09-remote => 09-service}/data_sink.py |   0
 test/{09-remote => 09-service}/dtypes.py    |   0
 test/09-service/model.py                    |  45 +++++++
 test/09-service/run_me.py                   |  86 +++++++++++++
 test/{09-remote => 09-service}/sensor.py    |   0
 test/remote_service.py                      |  43 -------
 15 files changed, 187 insertions(+), 521 deletions(-)
 delete mode 100644 src/digitaltwin/remote/README.md
 delete mode 100644 src/digitaltwin/remote/client.py
 delete mode 100644 src/digitaltwin/remote/remote_service.py
 delete mode 100644 test/09-remote/README.md
 delete mode 100644 test/09-remote/model.py
 delete mode 100644 test/09-remote/run_me.py
 create mode 100644 test/09-service/README.md
 rename test/{09-remote => 09-service}/agent.py (100%)
 rename test/{09-remote => 09-service}/data_sink.py (100%)
 rename test/{09-remote => 09-service}/dtypes.py (100%)
 create mode 100644 test/09-service/model.py
 create mode 100644 test/09-service/run_me.py
 rename test/{09-remote => 09-service}/sensor.py (100%)
 delete mode 100644 test/remote_service.py

diff --git a/.gitignore b/.gitignore
index 0e48c0e..d2d421a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -223,3 +223,7 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# runtime residue from asyncflow / rhapsody test runs
+asyncflow.session.*/
+telemetry-output/
diff --git a/src/digitaltwin/remote/README.md b/src/digitaltwin/remote/README.md
deleted file mode 100644
index 053ad97..0000000
--- a/src/digitaltwin/remote/README.md
+++ /dev/null
@@ -1,99 +0,0 @@
-# Digital Twin Runtime – Remote‑as‑a‑Service (DT-aaS)
-
-This directory contains a minimal implementation that turns a local **Digital Twin Runtime** into a remote procedure‑call service.
-
-## What is this?
-
-The core `DigitalTwin` package lives in `src/digitaltwin` and exposes the `DTRuntime` object, which you normally instantiate and use **directly**.  With the code in this directory you can:
-
-* **Expose an instance of `DTRuntime` over the network** using ZeroMQ.
-* **Interact with the runtime from the client** via the lightweight `RemoteDTRuntime` class.
-* **Serialize user code (Agents, Investigators, Tasks)** with `cloudpickle` to
-  pass them to the service.
-
-## Architecture Overview
-
-```
-+-------------------+           +-------------------+
-|  Client (Python)  |  <==>      |  Remote Service   |
-|  RemoteDTRuntime  |            |  RemoteDTService  |
-+-------------------+           +----------+--------+
-                                              |
-                                              v
-                                 +-----------------------+
-                                 |  DTRuntime (local)   |
-                                 +-----------------------+
-```
-
-* **`RemoteDTService`** – a ZeroMQ `REP` socket that listens on provided address.  It receives requests encoded as a tiny JSON frame that looks like:
-
-```json
-{
-    "method": "add_task", 
-    "args": ["<b64‑pickled‑obj>", "<b64‑pickled‑obj>"],
-    "kwargs": {"key":"<b64-pickled-obj"}
-}
-```
-
-The service unpickles the arguments, invokes the corresponding method on the wrapped `DTRuntime`, and returns a pickled result.
-
-* **`RemoteDTRuntime`** – a synchronous client that communicates with the
-  service over a ZeroMQ `REQ` socket.  It serializes your arguments with
-  `cloudpickle`, base64‑encodes them, and mirrors the remote API (`start`,
-  `add_task`, `add_investigator`, `add_agent`, etc....).
-
-* **`RemoteDTOrchestrator`** – a tiny helper that can create new
-  `RemoteDTRuntime` instances, should you want to launch multiple runtimes (sessions).
-
-## Getting Started
-
-1. **Install the project** (including the new `cloudpickle` dependency):
-
-   ```bash
-   pip install .
-   ```
-
-2. **Launch the remote service** from your application or a terminal:
-
-   ```bash
-   cd tests/
-   python3 remote_service
-   ```
-
-   The service will bind to `tcp://127.0.0.1:5555`.
-
-3. **Launch the PubSub broker**:
-
-   ```bash
-   cd tests/
-   python3 local_broker.py
-   ```
-
-3. **Remotely interact with runtime** from another process:
-
-   ```python
-   from digitaltwin.remote.client import RemoteDTRuntime
-
-   runtime = RemoteDTRuntime("tcp://localhost:5555")
-   # ... add tasks, reporters, etc.
-   runtime.start()
-   ```
-
-## Testing & Examples
-
-The test suite demonstrates both the server and client in action:
-
-* `test/09-remote/test_remote_service_client.py` – shows a client creating a
-  `RemoteDTRuntime`, invoking methods, and displaying the results
-* `test/remote_service.py` – runs a basic service using the `RemoteDTService` class.
-
-
-## Dependencies
-
-* `pyzmq` – ZeroMQ binding for Python.
-* `cloudpickle` – serialises arbitrary Python objects across the wire.
-
-
----
-
-The remote interface will eventually be ported over as an ORBIT plugin.
\ No newline at end of file
diff --git a/src/digitaltwin/remote/client.py b/src/digitaltwin/remote/client.py
deleted file mode 100644
index 3aedcc8..0000000
--- a/src/digitaltwin/remote/client.py
+++ /dev/null
@@ -1,136 +0,0 @@
-import zmq
-import base64
-import json
-import cloudpickle
-
-from digitaltwin.components import DataType
-import logging
-
-logger = logging.getLogger(__name__)
-
-###
-# See test/09-remote on how this works!
-###
-
-
-def register_user_modules(modules: list):
-    for module in modules:
-        cloudpickle.register_pickle_by_value(module)
-
-
-class RemoteDTRuntime:
-    """Proxy client for a :class:`RemoteDTService`.
-
-    The client serialises all arguments with ``cloudpickle`` and sends them
-    over a ZeroMQ REQ socket.  Responses are deserialised and returned.
-    It mirrors the public API of :class:`~digitaltwin.runtime.DTRuntime`.
-    """
-
-    def __init__(self, address: str):
-        self.ctx = zmq.Context.instance()
-        self.socket = self.ctx.socket(zmq.REQ)
-        self.socket.connect(address)
-
-        self._call("_new")
-
-    def package(self, module, *args, **kwargs):
-        cp_class = base64.b64encode(cloudpickle.dumps(module)).decode("ascii")
-        args_out = []
-        for a in args:
-            args_out.append(base64.b64encode(cloudpickle.dumps(a)).decode("ascii"))
-
-        kwargs_out = {}
-        for k, a in kwargs.items():
-            kwargs_out[k] = base64.b64encode(cloudpickle.dumps(a)).decode("ascii")
-
-        payload = {
-            "class": cp_class,
-            "args": args_out,
-            "kwargs": kwargs_out,
-            "pkg": "pkg",
-        }
-        logger.info(f"Package payload: {payload}")
-        identifier = self._register_artifact(payload)
-        return identifier
-
-    def _register_artifact(self, pkg):
-        identifier = self._call("_register_artifact", pkg)
-        return identifier
-
-    def _call(self, method: str, *args, **kwargs):
-        args_out = []
-        for a in args:
-            args_out.append(base64.b64encode(cloudpickle.dumps(a)).decode("ascii"))
-
-        kwargs_out = {}
-        for k, a in kwargs.items():
-            kwargs_out[k] = base64.b64encode(cloudpickle.dumps(a)).decode("ascii")
-
-        payload = json.dumps(
-            {"method": method, "args": args_out, "kwargs": kwargs_out}
-        ).encode("utf-8")
-        self.socket.send(payload)
-        resp = self.socket.recv()
-        return cloudpickle.loads(resp)
-
-    # --------------------------------------------------------------
-    # Wrapped methods – all sync to match the real DTRuntime API.
-    # --------------------------------------------------------------
-
-    def start(self):
-        return self._call("start")
-
-    def add_task(
-        self,
-        task_pkg,
-        input_dtype: DataType,
-        output_dtype: DataType,
-        is_persistent: bool = False,
-    ):
-        return self._call(
-            "add_task", task_pkg, input_dtype, output_dtype, is_persistent
-        )
-
-    def add_investigator(
-        self, inv_pkg, input_dtype: DataType, output_dtype: DataType, *args, **kwargs
-    ):
-        return self._call(
-            "add_investigator", inv_pkg, input_dtype, output_dtype, *args, **kwargs
-        )
-
-    def add_agent(
-        self, agent_pkg, input_dtype: DataType, output_dtype: DataType, *args, **kwargs
-    ):
-        return self._call(
-            "add_agent", agent_pkg, input_dtype, output_dtype, *args, **kwargs
-        )
-
-    def print_graph(self):
-        return self._call("print_graph")
-
-    def end(self):
-        # nop currently
-        self._call("_end")
-
-    def close(self):
-        self.socket.close()
-
-
-# Lightweight orchestrator that can create new RemoteDTRuntime sessions.
-class RemoteDTOrchestrator:
-    """Creates and manages RemoteDTRuntime sessions.
-
-    For now it simply instantiates a new :class:`RemoteDTRuntime` with a
-    configurable address.  The class can be extended to handle session
-    pooling, authentication, or reconnection logic.
-    """
-
-    def __init__(self, address: str):
-        self.address = address
-
-    def new_session(self) -> RemoteDTRuntime:
-        return RemoteDTRuntime(self.address)
-
-
-# NOTE: No additional logic is required – the real RemoteDTService
-# performs the round‑trip and forwards to the local DTRuntime.
diff --git a/src/digitaltwin/remote/remote_service.py b/src/digitaltwin/remote/remote_service.py
deleted file mode 100644
index 843f947..0000000
--- a/src/digitaltwin/remote/remote_service.py
+++ /dev/null
@@ -1,117 +0,0 @@
-import asyncio
-import base64
-import json
-import zmq
-import zmq.asyncio
-import cloudpickle
-from radical.asyncflow import WorkflowEngine  # type: ignore
-from ..runtime import DTRuntime
-from ..streaming import PubSubClient
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-###
-# See test/09-remote on how this works!
-###
-
-
-class RemoteDTService:
-    """ZeroMQ based service that forwards **synchronously** to a local
-    :class:`~digitaltwin.runtime.DTRuntime` instance.
-
-    All calls receive base64‑encoded cloudpickle payloads.  Errors are
-    returned as a pickled string describing the exception.
-    """
-
-    def __init__(self, flow: WorkflowEngine, bind_addr: str, streamer: PubSubClient):
-        self.bind_addr = bind_addr
-        self.ctx = zmq.asyncio.Context.instance()
-        self.socket = self.ctx.socket(zmq.REP)
-        self.socket.bind(self.bind_addr)
-
-        # support just one session / runtime right now...
-        self.runtime = None
-
-        self.flow = flow
-        self.streamer = streamer
-        self.artifacts = []
-
-    def _process_call(self, req: dict):
-        method = req["method"]
-        args_raw = req.get("args", [])
-        kwargs_raw = req.get("kwargs", [])
-
-        args = [cloudpickle.loads(base64.b64decode(arg)) for arg in args_raw]
-        kwargs = {
-            k: cloudpickle.loads(base64.b64decode(kwargs_raw[k])) for k in kwargs_raw
-        }
-
-        if method == "_new":
-            self.runtime = DTRuntime(self.flow, self.streamer)
-            return "ok"
-
-        if method == "print_graph":
-            return self.runtime.print_graph()
-
-        if method == "start":
-            return self.runtime.start()
-
-        if method == "_end":
-            return False
-
-        if method == "_register_artifact":
-            # try unpickling
-            package = args[0]
-            assert package["pkg"] == "pkg"
-
-            args_out = []
-            for a in package["args"]:
-                args_out.append(cloudpickle.loads(base64.b64decode(a)))
-
-            kwargs_out = {}
-            for k, a in package["kwargs"].items():
-                kwargs_out[k] = cloudpickle.loads(base64.b64decode(a))
-
-            class_comp = cloudpickle.loads(base64.b64decode(package["class"]))
-            logger.info(f"Instance creation: {class_comp}, {args_out}, {kwargs_out}")
-            obj = class_comp(self.flow, *args_out, **kwargs_out)
-
-            self.artifacts.append(obj)
-            return len(self.artifacts) - 1
-
-        # others require unpickle package
-        pkg = args[0]
-        logger.info(f"Call: {method}: {self.artifacts[pkg]}, {args[1:]}, {kwargs}")
-
-        if not hasattr(self.runtime, method):
-            return f"unknown method {method}"
-        fn = getattr(self.runtime, method)
-
-        # execute!
-        fn(self.artifacts[pkg], *args[1:], **kwargs)
-        return "ok"
-
-    async def _handle_request(self, msg: bytes) -> bytes | bool:
-        try:
-            req = json.loads(msg.decode("utf-8"))
-        except Exception as exc:  # bad framing
-            return cloudpickle.dumps(f"bad request: {exc}")
-
-        if req["method"] == "_end":
-            return False
-
-        output = self._process_call(req)
-
-        return cloudpickle.dumps(output)
-
-    async def serve(self):
-        while True:
-            msg = await self.socket.recv()
-            resp = await self._handle_request(msg)
-            if resp == False:
-                await self.socket.send(cloudpickle.dumps("ok"))
-                break
-            else:
-                await self.socket.send(resp)
diff --git a/test/09-remote/README.md b/test/09-remote/README.md
deleted file mode 100644
index e66a802..0000000
--- a/test/09-remote/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# Remote Demo: DT Runtime as a Service
-
-Digital Twin Application:
-- sensor counts to 10
-- 1 agent
-- data sink
-
-Client side:
-- `run_me.py` obtains a session (runtime on the server), submits an sensor,
-  agent, and sink to the remote service. 
-
-Server side:
-- `../remote_service.py` - simply serves as the runtime host. It cares not of the
-  actual app (hence outside of this directory). API calls are routed to the runtime
-- `../local_broker.py` - This is the pubsub broker backend. It is *not* know
-  anything about the runtime at all. 
-
-
-How this will relate to ORBIT:
-- Integration with ORBIT is taking this and packaging it into an ORBIT plugin.
-  The equivalent of the `../remote_service.py` and `../local_broker.py` will run
-  on the ORBIT broker host as a plugin. 
-- Currently, I'm using ZMQ for dummy transport, but this can easily be ported
-  over to ORBIT's transport utilities.
-- The client will run the equivalent of `run_me.py`  
diff --git a/test/09-remote/model.py b/test/09-remote/model.py
deleted file mode 100644
index b11b78f..0000000
--- a/test/09-remote/model.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import asyncio
-import os
-import sys
-
-from radical.asyncflow import WorkflowEngine
-from digitaltwin.components import ModelInvestigator, TypedData
-from digitaltwin.runtime import RuntimeAPI
-
-from dtypes import *
-import random
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class MyModel(ModelInvestigator):
-    def __init__(self, flow: WorkflowEngine, *args, **kwargs):
-        super().__init__(flow)
-        self.flow = flow
-
-        # no learning.... just inference for now.
-        # inference changes given the args passed in.
-
-        @self.flow.function_task
-        async def do_inference(in_data: TypedData, offset=1):
-            return TypedData(INFERENCE_DTYPE, offset - in_data.data)
-
-        self.inference_task = do_inference
-
-    async def main_loop(self, runtime: RuntimeAPI):
-        # runtime
-        runtime.set_inference_task(self.inference_task)
-
-        offset = 2
-        while True:
-            await asyncio.sleep(5)
-            runtime.publish_new_model({"offset": offset})
-            offset += 1
diff --git a/test/09-remote/run_me.py b/test/09-remote/run_me.py
deleted file mode 100644
index 35ac51f..0000000
--- a/test/09-remote/run_me.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import os
-import time
-
-
-from digitaltwin.components import TRUTHY, NULL_DTYPE
-from digitaltwin.remote.client import *
-
-from dtypes import *
-from sensor import MySensor
-from agent import MyAgent
-from data_sink import MySink
-
-# include all user code for registration!!!
-import dtypes
-import sensor
-import agent
-import model
-import data_sink
-
-register_user_modules([dtypes, sensor, agent, data_sink, model])
-
-from radical.asyncflow.logging import init_default_logger
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-# put it all together
-# sensor --> model --> data_sink
-
-
-def main():
-    init_default_logger(logging.INFO)
-    logging.getLogger("radical.asyncflow").setLevel(logging.WARNING)
-    logging.getLogger("rhapsody").setLevel(logging.WARNING)
-
-    address = os.environ.get("DT_REMOTE_SERVICE_ADDR", "tcp://127.0.0.1:5555")
-    remote = RemoteDTOrchestrator(address=address)
-
-    runtime = remote.new_session()
-
-    sensor_pkg = runtime.package(MySensor, 0, kwargs={})
-    agent_pkg = runtime.package(MyAgent, 0, kwargs={})
-    sink_pkg = runtime.package(MySink, 0, kwargs={})
-
-    runtime.add_task(sensor_pkg, TRUTHY, SENSOR_DTYPE, is_persistent=True)
-    runtime.add_agent(agent_pkg, SENSOR_DTYPE, INFERENCE_DTYPE)
-    runtime.add_task(sink_pkg, INFERENCE_DTYPE, NULL_DTYPE)
-
-    out = runtime.print_graph()
-    print(out)
-
-    runtime.start()
-
-    time.sleep(10)
-
-    runtime.end()
-    runtime.close()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/test/09-service/README.md b/test/09-service/README.md
new file mode 100644
index 0000000..4c5c293
--- /dev/null
+++ b/test/09-service/README.md
@@ -0,0 +1,52 @@
+# Service Demo: a digital twin on the ORBIT `dt` plugin
+
+Digital twin application:
+- sensor counts to 10 (a persistent component, running inline on the
+  service loop and publishing through its injected stream client)
+- 1 agent with one investigator
+- data sink
+
+Everything except `run_me.py` runs in the service: the client ships the
+component *classes* (cloudpickle) and drives the twin with short verbs.
+
+## Running it
+
+Three terminals.  All of them need `pip install .[service]` and the
+ORBIT broker cert/token in `~/.radical/orbit/`.
+
+```sh
+# 1 - the ORBIT broker, hosting the dt plugin
+radical-orbit-broker.py --plugins default,dt
+
+# 2 - a rhapsody endpoint: this is where the twin's tasks execute
+RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0 \
+RADICAL_ORBIT_RHAPSODY_BACKEND=concurrent \
+radical-orbit-endpoint.py -n dt_task_ep
+
+# 3 - the client
+cd test/09-service && python run_me.py
+```
+
+`DT_TASK_ENDPOINT` names the endpoint the twin's tasks go to (unset:
+ORBIT picks the first endpoint advertising rhapsody).  `DT_SERVICE_HOST`
+names the participant hosting the `dt` plugin (default `broker`; set it
+to an endpoint name for the endpoint-hosted deployment).
+
+The stream broker is *not* started by hand here: the plugin owns an
+embedded one, on a random loopback port, supervised and shared by every
+twin.
+
+## Reattaching
+
+The session is persistent and the sid is a bearer capability, so a
+client that goes away can come back to its still-running twins:
+
+```python
+dt = runtime.get_plugin('broker', 'dt', sid='session.xxxxxxxx')
+print(dt.twin_list())
+```
+
+`dt.admin_sessions()` lists every session on the service — owner, age,
+twins, states and last errors — which is how orphaned sessions are found
+and then torn down with the ordinary `twin_close` /
+`unregister_session` routes.
diff --git a/test/09-remote/agent.py b/test/09-service/agent.py
similarity index 100%
rename from test/09-remote/agent.py
rename to test/09-service/agent.py
diff --git a/test/09-remote/data_sink.py b/test/09-service/data_sink.py
similarity index 100%
rename from test/09-remote/data_sink.py
rename to test/09-service/data_sink.py
diff --git a/test/09-remote/dtypes.py b/test/09-service/dtypes.py
similarity index 100%
rename from test/09-remote/dtypes.py
rename to test/09-service/dtypes.py
diff --git a/test/09-service/model.py b/test/09-service/model.py
new file mode 100644
index 0000000..676d8bd
--- /dev/null
+++ b/test/09-service/model.py
@@ -0,0 +1,45 @@
+import asyncio
+import logging
+
+from radical.asyncflow import WorkflowEngine
+from digitaltwin.components import ModelInvestigator, TypedData
+from digitaltwin.runtime import RuntimeAPI
+
+from dtypes import *
+
+logger = logging.getLogger(__name__)
+
+
+class MyModel(ModelInvestigator):
+    """No learning -- just inference, whose result changes with the
+    published model arguments.
+
+    The compute runs on the engine (and therefore on a rhapsody
+    endpoint); the `TypedData` wrapping happens here, in the service.
+    Function task *arguments* are cloudpickled, but return values only
+    survive the ORBIT rhapsody plugin if they are JSON-safe or bytes --
+    so tasks return plain values.
+    """
+
+    def __init__(self, flow: WorkflowEngine, *args, **kwargs):
+        super().__init__(flow)
+        self.flow = flow
+
+        @self.flow.function_task
+        async def compute(in_data: TypedData, offset=1):
+            return offset - in_data.data
+
+        self.compute = compute
+
+    async def main_loop(self, runtime: RuntimeAPI):
+        async def do_inference(in_data: TypedData, offset=1):
+            return TypedData(INFERENCE_DTYPE, await self.compute(in_data,
+                                                                 offset=offset))
+
+        runtime.set_inference_task(do_inference)
+
+        offset = 2
+        while True:
+            runtime.publish_new_model({"offset": offset})
+            offset += 1
+            await asyncio.sleep(5)
diff --git a/test/09-service/run_me.py b/test/09-service/run_me.py
new file mode 100644
index 0000000..0499459
--- /dev/null
+++ b/test/09-service/run_me.py
@@ -0,0 +1,86 @@
+"""Integration demo: a digital twin hosted by the `dt` ORBIT plugin.
+
+    sensor --> agent (one investigator) --> data sink
+
+Everything below the client is remote: the twin's runtime, its stream
+client and the components all live in the plugin host; the components'
+compute runs on a rhapsody endpoint.  See README.md for the two services
+this needs.
+"""
+
+import json
+import logging
+import os
+import time
+
+from radical.orbit import EndpointRuntime
+
+from digitaltwin.components import NULL_DTYPE, TRUTHY
+from digitaltwin.service import register_user_modules
+
+from dtypes import INFERENCE_DTYPE, SENSOR_DTYPE
+from sensor import MySensor
+from agent import MyAgent
+from data_sink import MySink
+
+# the service has none of this code -- ship it by value
+import agent
+import data_sink
+import dtypes
+import model
+import sensor
+
+register_user_modules([dtypes, sensor, agent, model, data_sink])
+
+logger = logging.getLogger(__name__)
+
+# where the `dt` plugin is hosted ('broker', or an endpoint name), and
+# which endpoint runs the twin's tasks (unset: let ORBIT pick one)
+DT_HOST = os.environ.get("DT_SERVICE_HOST", "broker")
+TASK_ENDPOINT = os.environ.get("DT_TASK_ENDPOINT") or None
+
+# engine wiring, stated explicitly: one 'task' engine per session, on a
+# co-located endpoint, with the concurrent backend
+ENGINES = {
+    "engines": {
+        "task": {"endpoint_name": TASK_ENDPOINT, "backends": ["concurrent"]}
+    }
+}
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger("radical.orbit").setLevel(logging.WARNING)
+
+    runtime = EndpointRuntime()
+    runtime.start(wait=True)
+
+    try:
+        dt = runtime.get_plugin(DT_HOST, "dt", config=ENGINES)
+        print(f"session: {dt.sid}  (reattach with this sid)")
+
+        # one asynchronous verb: returns as soon as the twin is
+        # registered, the helper polls twin_list until it is ready
+        twin = dt.create_twin()
+        print(f"twin: {twin}")
+
+        dt.add_task(twin, dt.package(MySensor), TRUTHY, SENSOR_DTYPE,
+                    is_persistent=True)
+        dt.add_agent(twin, dt.package(MyAgent), SENSOR_DTYPE, INFERENCE_DTYPE)
+        dt.add_task(twin, dt.package(MySink), INFERENCE_DTYPE, NULL_DTYPE)
+
+        print(json.dumps(dt.describe(twin), indent=2))
+
+        dt.start(twin)
+        time.sleep(15)
+
+        print(json.dumps(dt.twin(twin), indent=2))
+
+        dt.twin_close(twin)
+
+    finally:
+        runtime.stop()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/09-remote/sensor.py b/test/09-service/sensor.py
similarity index 100%
rename from test/09-remote/sensor.py
rename to test/09-service/sensor.py
diff --git a/test/remote_service.py b/test/remote_service.py
deleted file mode 100644
index 2570856..0000000
--- a/test/remote_service.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import asyncio
-from concurrent.futures import ProcessPoolExecutor
-from radical.asyncflow import WorkflowEngine
-from rhapsody.backends import ConcurrentExecutionBackend
-from digitaltwin.streaming import connect_stream_client
-
-from digitaltwin.remote.remote_service import RemoteDTService
-
-from radical.asyncflow.logging import init_default_logger
-import logging
-import os
-
-logger = logging.getLogger(__name__)
-
-
-SERVICE = os.environ.get("DT_REMOTE_SERVICE_ADDR", "tcp://127.0.0.1:5555")
-
-# put it all together
-# sensor --> model --> data_sink
-
-
-async def main():
-    init_default_logger(logging.INFO)
-    logging.getLogger("radical.asyncflow").setLevel(logging.WARNING)
-    logging.getLogger("rhapsody").setLevel(logging.WARNING)
-
-    # create engine
-    exe = await ConcurrentExecutionBackend(ProcessPoolExecutor())
-    flow = await WorkflowEngine.create(backend=exe)
-
-    # create the twin's namespaced stream client (this prototype hosts a
-    # single session, so a single namespace will do)
-    pubsub_client = await connect_stream_client("remote-service")
-
-    service = RemoteDTService(flow, SERVICE, pubsub_client)
-    logger.info(f"Starting RemoteDTService on {SERVICE}")
-    await service.serve()
-
-    await flow.shutdown()
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

From c5d0ab5e06f74f915966ae2110c08da3bece180c Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:45:45 +0200
Subject: [PATCH 16/19] Tests for the dt plugin: units plus a live-stack
 integration suite

Unit tests (no broker): forced-persistent sessions, sid-as-bearer
reattach, twin id and verb validation, the wire format's size and
version-skew guards, the persistent-`function_task` warning, and the
embedded stream broker -- including the supervisor respawning it on the
addresses it reported.

Integration tests spin a real stack as fixtures (ORBIT broker on 8031
hosting `dt`, co-located rhapsody endpoint with `backends=['concurrent']`
and the notify window at 0) and cover the plan's M1 item 9: inference
round trip, two concurrent twins whose identical dtype labels do not
cross-subscribe, independent teardown, twin churn with a leak assertion,
`twin_close` with an inference in flight, client disconnect and reattach
by sid, idempotent retries, start-after-stop and graph-verb-after-stop
errors, component crash surfacing as `failed` with a last error, the
admin listing, and an endpoint-hosted smoke test.  They skip when no
broker can be started.

Also: a closed twin now reports `closed` (keeping its last error) and
drops its runtime, and a rejected verb logs its traceback service-side.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/service/session.py  |  13 +-
 test/integration/conftest.py        | 267 +++++++++++++++++
 test/integration/test_dt_service.py | 437 ++++++++++++++++++++++++++++
 test/integration/twin_components.py | 120 ++++++++
 test/unit/test_service.py           | 320 ++++++++++++++++++++
 5 files changed, 1155 insertions(+), 2 deletions(-)
 create mode 100644 test/integration/conftest.py
 create mode 100644 test/integration/test_dt_service.py
 create mode 100644 test/integration/twin_components.py
 create mode 100644 test/unit/test_service.py

diff --git a/src/digitaltwin/service/session.py b/src/digitaltwin/service/session.py
index d07a2a0..baa349a 100644
--- a/src/digitaltwin/service/session.py
+++ b/src/digitaltwin/service/session.py
@@ -145,6 +145,12 @@ async def close(self, timeout: float = TWIN_STOP_TIMEOUT) -> None:
         except Exception as exc:
             log.warning("[dt] twin %s teardown: %s", self.twin_id, exc)
 
+        # keep the last error, drop everything else: a closed twin reports
+        # `closed`, and nothing of it is left to hold memory
+        if self.runtime is not None:
+            self._last_error = self.runtime.last_error or self._last_error
+        self.runtime = None
+        self.stream = None
         self._state = STATE_CLOSED
 
 
@@ -231,9 +237,12 @@ async def twin_call(
             ) from None
         except (RuntimeError, ValueError, AssertionError) as exc:
             # the runtime's own refusals (stopped graph, start-after-stop,
-            # bad dtypes) are client errors, not service faults
+            # bad dtypes) are client errors, not service faults -- but the
+            # traceback still belongs in the service log
+            log.warning("[dt] twin %s: %s failed", twin_id, verb, exc_info=exc)
             raise HTTPException(
-                status_code=409, detail=f"twin {twin_id}: {verb}: {exc}"
+                status_code=409,
+                detail=f"twin {twin_id}: {verb}: {type(exc).__name__}: {exc}",
             ) from exc
 
         return {**self._twin_state(twin), **(extra or {})}
diff --git a/test/integration/conftest.py b/test/integration/conftest.py
new file mode 100644
index 0000000..1c0d6ff
--- /dev/null
+++ b/test/integration/conftest.py
@@ -0,0 +1,267 @@
+"""A live DTaaS stack for the integration tests.
+
+Session-scoped fixtures bring up, on loopback:
+
+- an ORBIT broker hosting the `dt` plugin (`--plugins default,dt`),
+- a co-located rhapsody endpoint with `backends=['concurrent']` and the
+  notification window at 0 (P2 -- otherwise every sequential task pays
+  250 ms),
+- a consumer runtime the tests get `DTClient`s from.
+
+Everything is skipped when the broker cannot be started (no certs, no
+token, port taken), so the suite stays runnable without a deployment.
+"""
+
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import time
+import uuid
+
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("radical.orbit")
+
+import httpx  # noqa: E402  (after the importorskip)
+
+from radical.orbit import EndpointRuntime  # noqa: E402
+
+REPO = Path(__file__).resolve().parents[2]
+SRC = REPO / "src"
+
+BROKER_HOST = "127.0.0.1"
+BROKER_PORT = int(os.environ.get("DT_TEST_BROKER_PORT", "8031"))
+BROKER_URL = f"https://{BROKER_HOST}:{BROKER_PORT}"
+
+TASK_ENDPOINT = "dt_test_task_ep"
+DT_ENDPOINT = "dt_test_dt_ep"  # endpoint-hosted `dt`, for the smoke test
+
+STARTUP_TIMEOUT = 60.0
+LOGS = Path(os.environ.get("DT_TEST_LOG_DIR", "/tmp")) / "dt-integration-logs"
+
+# engine wiring every test uses: the co-located endpoint, concurrent backend
+ENGINES = {
+    "engines": {
+        "task": {"endpoint_name": TASK_ENDPOINT, "backends": ["concurrent"]}
+    }
+}
+
+
+def _port_free(port: int) -> bool:
+    with socket.socket() as sock:
+        sock.settimeout(0.5)
+        return sock.connect_ex((BROKER_HOST, port)) != 0
+
+
+def _child_env(**extra: str) -> dict:
+    """Environment for a broker / endpoint child process.
+
+    `src` goes first on PYTHONPATH so the children run the working tree,
+    not whatever `digitaltwin` happens to be installed.
+    """
+
+    env = dict(os.environ)
+    env["PYTHONPATH"] = os.pathsep.join(
+        [str(SRC), *filter(None, [env.get("PYTHONPATH")])]
+    )
+    env["RADICAL_ORBIT_BROKER_URL"] = BROKER_URL
+    env.update(extra)
+
+    return env
+
+
+def _spawn(name: str, argv: list, **env: str) -> subprocess.Popen:
+    LOGS.mkdir(parents=True, exist_ok=True)
+    log = (LOGS / f"{name}.log").open("w")
+
+    return subprocess.Popen(
+        argv, env=_child_env(**env), stdout=log, stderr=subprocess.STDOUT
+    )
+
+
+def _terminate(proc: subprocess.Popen, timeout: float = 15.0) -> None:
+    if proc.poll() is not None:
+        return
+
+    proc.terminate()
+    try:
+        proc.wait(timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait(5)
+
+
+def _orbit_script(name: str) -> list:
+    """Locate an ORBIT entry script (installed, or in a source checkout)."""
+
+    import radical.orbit
+
+    candidates = [
+        Path(sys.executable).parent / name,  # this interpreter's env
+        # a source checkout: .../site-packages/radical/orbit -> ../../../bin
+        Path(radical.orbit.__file__).resolve().parents[3] / "bin" / name,
+    ]
+    for script in candidates:
+        if script.exists():
+            return [sys.executable, str(script)]
+
+    installed = shutil.which(name)
+    if installed:
+        return [installed]
+
+    pytest.skip(f"cannot locate {name}")
+
+
+@pytest.fixture(scope="session")
+def broker():
+    """An ORBIT broker on a non-default loopback port, hosting `dt`."""
+
+    if not _port_free(BROKER_PORT):
+        pytest.skip(f"port {BROKER_PORT} is busy")
+
+    argv = _orbit_script("radical-orbit-broker.py") + [
+        "--host", BROKER_HOST,
+        "--port", str(BROKER_PORT),
+        "--plugins", "default,dt",
+    ]
+    proc = _spawn("broker", argv)
+
+    try:
+        _await_broker(proc)
+        yield BROKER_URL
+    finally:
+        _terminate(proc)
+
+
+def _await_broker(proc: subprocess.Popen) -> None:
+    deadline = time.time() + STARTUP_TIMEOUT
+
+    while time.time() < deadline:
+        if proc.poll() is not None:
+            pytest.skip(f"broker exited early -- see {LOGS / 'broker.log'}")
+        try:
+            httpx.get(BROKER_URL + "/", verify=False, timeout=2)
+            return
+        except Exception:
+            time.sleep(0.25)
+
+    _terminate(proc)
+    pytest.skip(f"broker did not come up -- see {LOGS / 'broker.log'}")
+
+
+@pytest.fixture(scope="session")
+def task_endpoint(broker):
+    """A co-located rhapsody endpoint: where the twins' tasks execute."""
+
+    argv = _orbit_script("radical-orbit-endpoint.py") + [
+        "-n", TASK_ENDPOINT, "-u", broker, "-p", "default",
+    ]
+    proc = _spawn(
+        TASK_ENDPOINT,
+        argv,
+        RADICAL_ORBIT_RHAPSODY_BACKEND="concurrent",
+        RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW="0",
+    )
+
+    try:
+        _await_plugin(TASK_ENDPOINT, "rhapsody", proc)
+        yield TASK_ENDPOINT
+    finally:
+        _terminate(proc)
+
+
+@pytest.fixture(scope="session")
+def dt_endpoint(broker):
+    """An endpoint hosting the `dt` plugin itself (endpoint-hosted mode).
+
+    It deliberately does *not* load rhapsody: its twins' tasks go to
+    `task_endpoint`, exactly as in the broker-hosted deployment.
+    """
+
+    argv = _orbit_script("radical-orbit-endpoint.py") + [
+        "-n", DT_ENDPOINT, "-u", broker, "-p", "dt",
+    ]
+    proc = _spawn(DT_ENDPOINT, argv)
+
+    try:
+        _await_plugin(DT_ENDPOINT, "dt", proc)
+        yield DT_ENDPOINT
+    finally:
+        _terminate(proc)
+
+
+def _await_plugin(endpoint: str, plugin: str, proc: subprocess.Popen) -> None:
+    """Wait until `endpoint` advertises `plugin` in the broker topology."""
+
+    runtime = EndpointRuntime(broker_url=BROKER_URL)
+    runtime.start(wait=True)
+
+    try:
+        deadline = time.time() + STARTUP_TIMEOUT
+        while time.time() < deadline:
+            if proc.poll() is not None:
+                pytest.skip(f"{endpoint} exited -- see {LOGS / endpoint}.log")
+
+            info = runtime.topology().get(endpoint) or {}
+            if plugin in (info.get("plugins") or {}):
+                return
+
+            time.sleep(0.25)
+
+    finally:
+        runtime.stop()
+
+    _terminate(proc)
+    pytest.skip(f"{endpoint} never advertised {plugin!r}")
+
+
+@pytest.fixture(scope="session")
+def stack(broker, task_endpoint):
+    """The full broker + endpoint stack; returns the broker URL."""
+
+    return broker
+
+
+@pytest.fixture
+def runtime(stack):
+    """A fresh consumer runtime -- one per test, like a real client."""
+
+    rt = EndpointRuntime(broker_url=stack)
+    rt.start(wait=True)
+    try:
+        yield rt
+    finally:
+        rt.stop()
+
+
+@pytest.fixture
+def dt(runtime):
+    """A `DTClient` on a fresh session, torn down with the test."""
+
+    client = runtime.get_plugin("broker", "dt", config=ENGINES)
+    try:
+        yield client
+    finally:
+        _drop_session(client)
+
+
+def _drop_session(client) -> None:
+    """Close every twin, then unregister -- sessions are immortal."""
+
+    try:
+        for twin in client.twin_list():
+            client.twin_close(twin["twin_id"])
+        client.unregister_session()
+    except Exception as exc:  # a test may have closed it already
+        print(f"session teardown: {exc}")
+
+
+@pytest.fixture
+def twin_id():
+    """A fresh client-supplied twin uuid."""
+
+    return str(uuid.uuid4())
diff --git a/test/integration/test_dt_service.py b/test/integration/test_dt_service.py
new file mode 100644
index 0000000..67958b4
--- /dev/null
+++ b/test/integration/test_dt_service.py
@@ -0,0 +1,437 @@
+"""Integration tests for the `dt` plugin against a live ORBIT stack.
+
+Covers the DTaaS plan's M1 item 9: two concurrent twins in one session,
+independent teardown, twin churn with a leak assertion, `twin_close`
+with an inference in flight, client disconnect -> reattach by sid,
+idempotent retries, and a component crash surfacing as `failed` plus a
+last error through `twin_list`.
+"""
+
+import asyncio
+import os
+import threading
+import time
+import uuid
+
+import pytest
+
+from radical.orbit import EndpointRuntime
+
+from digitaltwin.components import NULL_DTYPE, TRUTHY, TypedData
+from digitaltwin.service import register_user_modules
+from digitaltwin.streaming import connect_stream_client
+
+import twin_components
+
+from conftest import ENGINES, LOGS
+from twin_components import (
+    ECHO_DTYPE,
+    INFERENCE_DTYPE,
+    SENSOR_DTYPE,
+    CountingSensor,
+    CrashingTask,
+    EchoSink,
+    MisplacedFunctionTask,
+    OffsetModel,
+    SlowModel,
+)
+
+pytestmark = pytest.mark.integration
+
+register_user_modules([twin_components])
+
+POLL_TIMEOUT = 60.0
+
+
+# ---------------------------------------------------------------------------
+# helpers
+# ---------------------------------------------------------------------------
+
+def build_pipeline(dt, twin, offset=100):
+    """sensor -> investigator -> echo sink, the standard test twin."""
+
+    dt.add_task(twin, dt.package(CountingSensor), TRUTHY, SENSOR_DTYPE,
+                is_persistent=True)
+    dt.add_investigator(twin, dt.package(OffsetModel, offset=offset),
+                        SENSOR_DTYPE, INFERENCE_DTYPE)
+    dt.add_task(twin, dt.package(EchoSink), INFERENCE_DTYPE, NULL_DTYPE)
+
+
+def await_state(dt, twin, *states, timeout=POLL_TIMEOUT):
+    """Poll `twin_list` -- the only observation mechanism in v1."""
+
+    deadline = time.time() + timeout
+
+    while True:
+        entry = dt.twin(twin)
+        if entry["state"] in states:
+            return entry
+        if time.time() > deadline:
+            pytest.fail(f"twin {twin} stuck in {entry['state']}: {entry}")
+        time.sleep(0.25)
+
+
+def stream_addresses(dt):
+    """The service's embedded stream broker, from the admin listing."""
+
+    addrs = dt.admin_sessions()["stream_broker"]["addresses"]
+    assert addrs, "the service reports no stream broker"
+
+    return addrs
+
+
+async def collect(dt, twin, dtype, count, timeout=POLL_TIMEOUT):
+    """Subscribe to one twin's stream and collect `count` messages.
+
+    Loopback only -- the service and the tests share a host here.
+    """
+
+    pub_addr, sub_addr = stream_addresses(dt)
+    client = await connect_stream_client(twin, pub_addr, sub_addr)
+    queue: asyncio.Queue = asyncio.Queue()
+
+    try:
+        await client.subscribe_to_dtype(dtype, queue)
+        return [
+            (await asyncio.wait_for(queue.get(), timeout)).data
+            for _ in range(count)
+        ]
+    finally:
+        await client.close()
+
+
+def broker_fds():
+    """Open file descriptors of the broker process (Linux only)."""
+
+    pids = [p for p in os.listdir("/proc") if p.isdigit()]
+    for pid in pids:
+        try:
+            cmdline = open(f"/proc/{pid}/cmdline", "rb").read().decode()
+            if "radical-orbit-broker" in cmdline:
+                return len(os.listdir(f"/proc/{pid}/fd"))
+        except OSError:
+            continue
+    return None
+
+
+# ---------------------------------------------------------------------------
+# the happy path
+# ---------------------------------------------------------------------------
+
+def test_inference_roundtrip(dt, twin_id):
+    """A twin computes on the endpoint and answers a client's query."""
+
+    dt.create_twin(twin_id)
+    dt.add_investigator(twin_id, dt.package(OffsetModel, offset=7),
+                        SENSOR_DTYPE, INFERENCE_DTYPE)
+    assert dt.start(twin_id) == "running"
+
+    answer = dt.get_inference(twin_id, TypedData(SENSOR_DTYPE, 5),
+                              INFERENCE_DTYPE, timeout=120)
+
+    assert isinstance(answer, TypedData)
+    assert answer.data == 12
+
+    graph = dt.describe(twin_id)
+    assert graph["namespace"] == twin_id
+    assert graph["state"] == "running"
+
+
+def test_two_twins_one_session_are_independent(dt):
+    """Two twins, identical dtype labels, one session.
+
+    Their streams must not cross (topic namespacing), and closing one
+    must leave the other running.
+    """
+
+    first, second = str(uuid.uuid4()), str(uuid.uuid4())
+    bands = {first: 1_000_000, second: 2_000_000}
+
+    for twin, offset in bands.items():
+        dt.create_twin(twin)
+        build_pipeline(dt, twin, offset=offset)
+        dt.start(twin)
+
+    seen_first = asyncio.run(collect(dt, first, ECHO_DTYPE, 3))
+    seen_second = asyncio.run(collect(dt, second, ECHO_DTYPE, 3))
+
+    # same dtype label, different namespace: no cross-subscription
+    def in_band(values, base):
+        return all(base <= v < base + 100_000 for v in values)
+
+    assert in_band(seen_first, bands[first]), seen_first
+    assert in_band(seen_second, bands[second]), seen_second
+
+    assert dt.twin_close(first) == "closed"
+
+    # the sibling is untouched and still producing
+    assert dt.twin(second)["state"] == "running"
+    assert asyncio.run(collect(dt, second, ECHO_DTYPE, 2))
+
+    assert {t["twin_id"] for t in dt.twin_list()} == {second}
+
+
+# ---------------------------------------------------------------------------
+# lifecycle
+# ---------------------------------------------------------------------------
+
+def test_twin_churn_leaks_nothing(dt):
+    """Create/close cycles must not accumulate twins, engines or fds."""
+
+    def cycle():
+        twin = str(uuid.uuid4())
+        dt.create_twin(twin)
+        build_pipeline(dt, twin)
+        dt.start(twin)
+        await_state(dt, twin, "running")
+        dt.twin_close(twin)
+
+    cycle()  # first cycle pays the engine + stream broker startup
+    before = broker_fds()
+
+    for _ in range(4):
+        cycle()
+
+    assert dt.twin_list() == []
+
+    session = next(s for s in dt.admin_sessions()["sessions"]
+                   if s["sid"] == dt.sid)
+    # one engine per session, never one per twin
+    assert session["engines"] == ["task"]
+    assert session["twins"] == []
+
+    after = broker_fds()
+    if before is not None and after is not None:
+        assert after <= before + 8, f"broker fds {before} -> {after}"
+
+
+def test_twin_close_with_inference_in_flight(dt, twin_id):
+    """A `twin_close` must not be held up by -- nor strand -- a call in
+    flight; the caller gets a prompt, clear error."""
+
+    dt.create_twin(twin_id)
+    dt.add_investigator(twin_id, dt.package(SlowModel), SENSOR_DTYPE,
+                        INFERENCE_DTYPE)
+    dt.start(twin_id)
+
+    failure = {}
+    started = threading.Event()
+
+    def infer():
+        started.set()
+        try:
+            dt.get_inference(twin_id, TypedData(SENSOR_DTYPE, 1),
+                             INFERENCE_DTYPE, timeout=300)
+            failure["result"] = "returned unexpectedly"
+        except Exception as exc:
+            failure["error"] = str(exc)
+
+    caller = threading.Thread(target=infer, daemon=True)
+    caller.start()
+    started.wait(5)
+    time.sleep(2)  # let the inference reach the runtime
+
+    t0 = time.time()
+    assert dt.twin_close(twin_id) == "closed"
+    assert time.time() - t0 < 30, "twin_close waited for the inference"
+
+    caller.join(30)
+    assert not caller.is_alive()
+    assert "error" in failure, failure
+    assert "closed" in failure["error"], failure
+
+
+def test_idempotent_retries(dt, twin_id):
+    """The bearer-sid retry path: every verb a client may resend twice."""
+
+    dt.create_twin(twin_id)
+    # same uuid again: a no-op reporting current state, not a second twin
+    dt.create_twin(twin_id)
+    assert len(dt.twin_list()) == 1
+
+    dt.add_investigator(twin_id, dt.package(OffsetModel), SENSOR_DTYPE,
+                        INFERENCE_DTYPE)
+
+    assert dt.start(twin_id) == "running"
+    assert dt.start(twin_id) == "running"
+
+    assert dt.stop(twin_id) == "stopped"
+    assert dt.stop(twin_id) == "stopped"
+
+    assert dt.twin_close(twin_id) == "closed"
+    assert dt.twin_close(twin_id) == "closed"
+    assert dt.twin_list() == []
+
+
+def test_start_after_stop_is_an_error(dt, twin_id):
+    """`stop` is terminal in v1."""
+
+    dt.create_twin(twin_id)
+    dt.start(twin_id)
+    dt.stop(twin_id)
+
+    with pytest.raises(RuntimeError, match="terminal"):
+        dt.start(twin_id)
+
+
+def test_graph_verb_after_stop_is_a_clear_error(dt, twin_id):
+    """The runtime refuses graph changes on a stopped twin; the service
+    turns that into a 409 the client can read."""
+
+    dt.create_twin(twin_id)
+    dt.stop(twin_id)
+
+    with pytest.raises(RuntimeError, match="graph cannot be changed"):
+        dt.add_investigator(twin_id, dt.package(OffsetModel), SENSOR_DTYPE,
+                            INFERENCE_DTYPE)
+
+
+def test_component_crash_surfaces_as_failed(dt, twin_id):
+    """A dying component lands in the twin state, not in the log."""
+
+    dt.create_twin(twin_id)
+    dt.add_task(twin_id, dt.package(CrashingTask), TRUTHY, NULL_DTYPE,
+                is_persistent=True)
+    dt.start(twin_id)
+
+    entry = await_state(dt, twin_id, "failed")
+    assert "component crashed on purpose" in entry["last_error"]
+
+
+def test_unknown_twin_and_verb_are_rejected(dt):
+    with pytest.raises(RuntimeError, match="unknown twin"):
+        dt.start("no-such-twin")
+
+
+# ---------------------------------------------------------------------------
+# sessions
+# ---------------------------------------------------------------------------
+
+def test_client_disconnect_then_reattach_by_sid(stack, twin_id):
+    """Twins survive their client, and the sid gets them back."""
+
+    first = EndpointRuntime(broker_url=stack)
+    first.start(wait=True)
+    dt = first.get_plugin("broker", "dt", config=ENGINES)
+    sid = dt.sid
+
+    dt.create_twin(twin_id)
+    build_pipeline(dt, twin_id)
+    dt.start(twin_id)
+
+    # the client goes away entirely -- it comes back as a different
+    # participant, which is exactly what the bearer sid is for
+    first.stop()
+    time.sleep(2)
+
+    runtime = EndpointRuntime(broker_url=stack)
+    runtime.start(wait=True)
+    try:
+        again = runtime.get_plugin("broker", "dt", sid=sid)
+        assert again.sid == sid
+
+        entry = again.twin(twin_id)
+        assert entry["state"] == "running"
+
+        # and it is still doing work
+        assert asyncio.run(collect(again, twin_id, ECHO_DTYPE, 2))
+
+        again.twin_close(twin_id)
+        again.unregister_session()
+    finally:
+        runtime.stop()
+
+
+def test_admin_sessions_lists_twins_and_errors(dt, twin_id):
+    dt.create_twin(twin_id)
+    dt.add_task(twin_id, dt.package(CrashingTask), TRUTHY, NULL_DTYPE,
+                is_persistent=True)
+    dt.start(twin_id)
+    await_state(dt, twin_id, "failed")
+
+    listing = dt.admin_sessions()
+    session = next(s for s in listing["sessions"] if s["sid"] == dt.sid)
+
+    assert session["lifetime"] == "persistent"  # forced by the plugin
+    assert session["age"] >= 0
+    assert "owner" in session
+
+    twin = next(t for t in session["twins"] if t["twin_id"] == twin_id)
+    assert twin["state"] == "failed"
+    assert "component crashed on purpose" in twin["last_error"]
+
+    assert listing["stream_broker"]["alive"] is True
+
+
+def test_twin_ids_are_globally_unique(dt, runtime, twin_id):
+    """Twin ids namespace a plugin-wide stream broker, so a second
+    session may not reuse a live one."""
+
+    dt.create_twin(twin_id)
+
+    other = runtime.get_plugin("broker", "dt", config=ENGINES)
+    try:
+        with pytest.raises(RuntimeError, match="already used"):
+            other.create_twin(twin_id)
+    finally:
+        other.unregister_session()
+
+
+# ---------------------------------------------------------------------------
+# guards
+# ---------------------------------------------------------------------------
+
+def test_persistent_function_task_warns(dt, twin_id):
+    """The service-side guard for the actual migration mistake."""
+
+    dt.create_twin(twin_id)
+    dt.add_task(twin_id, dt.package(MisplacedFunctionTask), TRUTHY,
+                NULL_DTYPE, is_persistent=True)
+
+    log = (LOGS / "broker.log").read_text()
+    assert "MisplacedFunctionTask registered 1 function_task" in log
+
+
+def test_version_skew_is_rejected(dt, twin_id):
+    """A payload from a skewed interpreter must not reach cloudpickle."""
+
+    dt.create_twin(twin_id)
+
+    resp = dt._request(
+        "POST",
+        dt._url(f"twin_call/{dt.sid}/{twin_id}"),
+        json={"verb": "start", "payload": "", "client": {
+            "python": "2.7", "cloudpickle": "0.1"}},
+    )
+
+    assert resp.status_code == 400, resp.text
+    assert "version skew" in resp.text
+
+
+# ---------------------------------------------------------------------------
+# endpoint-hosted deployment
+# ---------------------------------------------------------------------------
+
+def test_endpoint_hosted_smoke(dt_endpoint, task_endpoint, runtime):
+    """The endpoint-hosted mode must not silently rot.
+
+    Create / list / close only -- `get_inference` is the one verb the
+    30 s relay backstop can cut short until P0 lands.
+    """
+
+    dt = runtime.get_plugin(dt_endpoint, "dt", config=ENGINES)
+    twin = str(uuid.uuid4())
+
+    try:
+        dt.create_twin(twin)
+        assert [t["twin_id"] for t in dt.twin_list()] == [twin]
+
+        dt.add_investigator(twin, dt.package(OffsetModel), SENSOR_DTYPE,
+                            INFERENCE_DTYPE)
+        assert dt.start(twin) == "running"
+
+        assert dt.twin_close(twin) == "closed"
+        assert dt.twin_list() == []
+
+    finally:
+        dt.unregister_session()
diff --git a/test/integration/twin_components.py b/test/integration/twin_components.py
new file mode 100644
index 0000000..630876c
--- /dev/null
+++ b/test/integration/twin_components.py
@@ -0,0 +1,120 @@
+"""Twin components the integration tests ship to the service.
+
+This module is cloudpickled by value (the service has no copy of it), so
+it must stay importable on its own -- no test imports, no fixtures.
+"""
+
+import asyncio
+
+from digitaltwin.components import (
+    DataType,
+    ModelInvestigator,
+    TypedData,
+    UtilityTask,
+)
+
+SENSOR_DTYPE = DataType("sensor")
+INFERENCE_DTYPE = DataType("inference")
+ECHO_DTYPE = DataType("echo")
+
+
+class CountingSensor(UtilityTask):
+    """Persistent source: publishes 0, 1, 2, ... into the twin's stream.
+
+    Plain async code on the service loop with the injected stream client
+    -- the persistent-component contract.
+    """
+
+    def __init__(self, flow, count: int = 10_000, interval: float = 0.2):
+        super().__init__(flow)
+        self.count = count
+        self.interval = interval
+
+    async def main_loop(self, runtime, in_data):
+        for value in range(self.count):
+            await asyncio.sleep(self.interval)
+            await runtime.stream.publish(SENSOR_DTYPE, value)
+
+
+class OffsetModel(ModelInvestigator):
+    """Inference on the engine: `sensor value + offset`.
+
+    Note the split: the *task* returns a plain value and the component
+    wraps it in `TypedData`.  Task arguments keep full cloudpickle
+    fidelity, but return values only survive the ORBIT rhapsody plugin
+    when they are JSON-safe (or `bytes`) -- see README.
+    """
+
+    def __init__(self, flow, offset: int = 100):
+        super().__init__(flow)
+        self.offset = offset
+
+        @flow.function_task
+        async def compute(in_data: TypedData, offset: int = 0):
+            return in_data.data + offset
+
+        self.compute = compute
+
+    async def main_loop(self, runtime):
+        async def infer(in_data: TypedData, offset: int = 0):
+            value = await self.compute(in_data, offset=offset)
+            return TypedData(INFERENCE_DTYPE, value)
+
+        runtime.set_inference_task(infer)
+        runtime.publish_new_model({"offset": self.offset})
+
+
+class SlowModel(ModelInvestigator):
+    """Inference that never finishes in test time -- for the teardown
+    path of a `twin_close` with a call in flight."""
+
+    def __init__(self, flow, delay: float = 300.0):
+        super().__init__(flow)
+        self.delay = delay
+
+    async def main_loop(self, runtime):
+        async def infer(in_data: TypedData, **kwargs):
+            await asyncio.sleep(self.delay)
+            return TypedData(INFERENCE_DTYPE, in_data.data)
+
+        runtime.set_inference_task(infer)
+        runtime.publish_new_model({})
+
+
+class EchoSink(UtilityTask):
+    """Terminal component: republishes what it receives, so a client can
+    watch the pipeline over the twin's own (namespaced) stream."""
+
+    async def main_loop(self, runtime, in_data):
+        await runtime.stream.publish(ECHO_DTYPE, in_data.data)
+
+
+class CrashingTask(UtilityTask):
+    """Persistent component that dies -- the twin must land in `failed`
+    with the reason visible in `twin_list`."""
+
+    def __init__(self, flow, delay: float = 0.1):
+        super().__init__(flow)
+        self.delay = delay
+
+    async def main_loop(self, runtime, in_data):
+        await asyncio.sleep(self.delay)
+        raise RuntimeError("component crashed on purpose")
+
+
+class MisplacedFunctionTask(UtilityTask):
+    """A persistent component written the *wrong* way: its body is a
+    `function_task`, which would occupy a backend slot for the twin's
+    lifetime.  The service warns about exactly this."""
+
+    def __init__(self, flow):
+        super().__init__(flow)
+
+        @flow.function_task
+        async def body():
+            return 1
+
+        self.body = body
+
+    async def main_loop(self, runtime, in_data):
+        await asyncio.sleep(3600)
diff --git a/test/unit/test_service.py b/test/unit/test_service.py
new file mode 100644
index 0000000..70a8fc3
--- /dev/null
+++ b/test/unit/test_service.py
@@ -0,0 +1,320 @@
+"""Unit tests for the `dt` plugin: policy, wire format, guards.
+
+No ORBIT broker involved -- routes are exercised over Starlette's
+`TestClient` (plugin route registration is dual), sessions and the
+stream-broker supervisor directly.
+"""
+
+import asyncio
+import sys
+
+import cloudpickle
+import pytest
+
+from fastapi import FastAPI
+from starlette.testclient import TestClient
+
+from digitaltwin.components import UtilityTask
+from digitaltwin.service.plugin import PluginDT
+from digitaltwin.service.session import DTSession, TwinInstance
+from digitaltwin.service.wire import (
+    MAX_PAYLOAD,
+    Package,
+    check_versions,
+    decode,
+    encode,
+    encode_checked,
+    version_stamp,
+)
+
+
+@pytest.fixture
+def plugin():
+    """A `PluginDT` on a bare app -- no broker, no engines."""
+
+    return PluginDT(FastAPI())
+
+
+@pytest.fixture
+def client(plugin):
+    return TestClient(plugin._app)
+
+
+# ---------------------------------------------------------------------------
+# session policy
+# ---------------------------------------------------------------------------
+
+def test_sessions_are_forced_persistent(plugin, client):
+    """Whatever the client asks for, twins must outlive it."""
+
+    for body in ({}, {"lifetime": "ephemeral"}, {"lifetime": "ttl", "ttl": 5}):
+        sid = client.post("/dt/register_session", json=body).json()["sid"]
+        record = plugin._records[sid]
+
+        assert record.lifetime == "persistent"
+        assert record.ttl is None
+
+
+def test_sid_is_a_bearer_capability(plugin, client):
+    """A reconnecting client is a different participant; it must still
+    get its own twins back."""
+
+    resp = client.post("/dt/register_session", json={},
+                       headers={"x-orbit-src": "client.1"})
+    sid = resp.json()["sid"]
+    assert plugin._records[sid].owner == "client.1"
+
+    again = client.post("/dt/register_session", json={"sid": sid},
+                        headers={"x-orbit-src": "client.2"})
+
+    assert again.status_code == 200
+    assert again.json() == {"sid": sid, "reattached": True}
+
+
+def test_register_session_rejects_a_bad_config(client):
+    resp = client.post("/dt/register_session", json={"config": "nope"})
+
+    assert resp.status_code == 400
+    assert "config" in resp.text
+
+
+def test_admin_sessions_reports_policy_and_broker(plugin, client):
+    sid = client.post("/dt/register_session", json={}).json()["sid"]
+    listing = client.get("/dt/admin/sessions").json()
+
+    entry = next(s for s in listing["sessions"] if s["sid"] == sid)
+    assert entry["lifetime"] == "persistent"
+    assert entry["twins"] == []
+    assert entry["engines"] == []
+
+    # nothing needed the stream broker yet, so none was started
+    assert listing["stream_broker"] == {"addresses": None, "alive": False}
+
+
+# ---------------------------------------------------------------------------
+# routes and validation
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize("twin_id", ["", "with/slash", "with space", None, 42])
+def test_twin_create_rejects_bad_ids(client, twin_id):
+    """A twin id is a stream namespace: a separator would let two twins
+    alias each other's topics."""
+
+    sid = client.post("/dt/register_session", json={}).json()["sid"]
+    resp = client.post(f"/dt/twin_create/{sid}", json={"twin_id": twin_id})
+
+    assert resp.status_code == 400
+
+
+def test_twin_call_rejects_unknown_verbs(client):
+    sid = client.post("/dt/register_session", json={}).json()["sid"]
+    resp = client.post(f"/dt/twin_call/{sid}/t1",
+                       json={"verb": "rm_rf", "payload": "", "client": {}})
+
+    assert resp.status_code == 400
+    assert "unknown verb" in resp.text
+
+
+def test_twin_call_rejects_version_skew(client):
+    sid = client.post("/dt/register_session", json={}).json()["sid"]
+    resp = client.post(
+        f"/dt/twin_call/{sid}/t1",
+        json={"verb": "start", "payload": "",
+              "client": {"python": "2.7", "cloudpickle": "0.1"}},
+    )
+
+    assert resp.status_code == 400
+    assert "version skew" in resp.text
+
+
+def test_twin_close_is_idempotent_for_unknown_twins(client):
+    sid = client.post("/dt/register_session", json={}).json()["sid"]
+    resp = client.post(f"/dt/twin_close/{sid}/never-existed")
+
+    assert resp.status_code == 200
+    assert resp.json()["state"] == "closed"
+
+
+def test_unknown_session_is_404(client):
+    assert client.get("/dt/twin_list/session.nope").status_code == 404
+
+
+# ---------------------------------------------------------------------------
+# wire format
+# ---------------------------------------------------------------------------
+
+def test_encode_roundtrip():
+    payload = {"args": (1, "two", [3]), "kwargs": {"k": {"nested": True}}}
+
+    assert decode(encode(payload)) == payload
+
+
+def test_size_check_refuses_oversized_payloads():
+    with pytest.raises(ValueError, match="frame cap"):
+        encode_checked(bytearray(MAX_PAYLOAD), "test payload")
+
+
+def test_size_check_passes_a_normal_payload():
+    assert encode_checked({"args": (1, 2)}, "test payload")
+
+
+def test_version_stamp_matches_itself():
+    check_versions(version_stamp())
+
+
+@pytest.mark.parametrize(
+    "stamp",
+    [
+        {},
+        {"python": "2.7", "cloudpickle": cloudpickle.__version__},
+        {"python": "%d.%d" % sys.version_info[:2], "cloudpickle": "0.1"},
+    ],
+)
+def test_version_skew_is_rejected(stamp):
+    with pytest.raises(ValueError):
+        check_versions(stamp)
+
+
+def test_package_instantiates_with_the_injected_engine():
+    class Component:
+        def __init__(self, flow, a, b=0):
+            self.flow, self.a, self.b = flow, a, b
+
+    component = Package(Component, (1,), {"b": 2}).instantiate("engine")
+
+    assert (component.flow, component.a, component.b) == ("engine", 1, 2)
+
+
+# ---------------------------------------------------------------------------
+# the persistent-component guard
+# ---------------------------------------------------------------------------
+
+class _FakeFlow:
+    """Just enough engine for `_instantiate`."""
+
+    def __init__(self):
+        self.registered = []
+
+    def function_task(self, func):
+        self.registered.append(func)
+        return func
+
+
+class _Persistent(UtilityTask):
+    def __init__(self, flow):
+        super().__init__(flow)
+
+        @flow.function_task
+        async def body():
+            return 1
+
+        self.body = body
+
+
+class _Plain(UtilityTask):
+    pass
+
+
+def _twin_with(flow):
+    twin = TwinInstance("t1")
+    twin.runtime = type("R", (), {"flow": flow})()
+
+    return twin
+
+
+def test_persistent_function_task_warns(caplog):
+    flow = _FakeFlow()
+    session = DTSession("s1")
+
+    with caplog.at_level("WARNING"):
+        session._instantiate(Package(_Persistent), _twin_with(flow),
+                             is_persistent=True)
+
+    assert "registered 1 function_task" in caplog.text
+    # the engine is handed back unpatched
+    assert flow.function_task.__func__ is _FakeFlow.function_task
+
+
+def test_non_persistent_function_task_is_fine(caplog):
+    session = DTSession("s1")
+
+    with caplog.at_level("WARNING"):
+        session._instantiate(Package(_Persistent), _twin_with(_FakeFlow()))
+
+    assert "function_task" not in caplog.text
+
+
+def test_persistent_without_function_tasks_is_fine(caplog):
+    session = DTSession("s1")
+
+    with caplog.at_level("WARNING"):
+        session._instantiate(Package(_Plain), _twin_with(_FakeFlow()),
+                             is_persistent=True)
+
+    assert "function_task" not in caplog.text
+
+
+def test_instantiate_rejects_a_non_package():
+    with pytest.raises(ValueError, match="package"):
+        DTSession("s1")._instantiate(_Plain, _twin_with(_FakeFlow()))
+
+
+# ---------------------------------------------------------------------------
+# the embedded stream broker and its supervisor
+# ---------------------------------------------------------------------------
+
+async def test_stream_broker_starts_on_loopback_once(plugin):
+    """One broker per plugin, shared by every twin, bound to loopback."""
+
+    try:
+        first = await plugin.stream_addresses()
+        second = await plugin.stream_addresses()
+
+        assert first == second
+        assert all(addr.startswith("tcp://127.0.0.1:") for addr in first)
+        assert plugin._stream_broker.is_alive()
+
+    finally:
+        await plugin.shutdown()
+
+
+async def test_supervisor_respawns_on_the_same_addresses(plugin, monkeypatch):
+    """A silently dead stream broker would stall every twin's stream."""
+
+    monkeypatch.setattr("digitaltwin.service.plugin.BROKER_WATCH_INTERVAL", 0.1)
+
+    try:
+        addrs = await plugin.stream_addresses()
+        first_pid = plugin._stream_broker._proc.pid
+
+        plugin._stream_broker._proc.kill()
+        await asyncio.wait_for(_respawned(plugin, first_pid), 20)
+
+        assert plugin._stream_broker.get_connection_str() == addrs
+        assert plugin._stream_broker.is_alive()
+
+    finally:
+        await plugin.shutdown()
+
+
+async def _respawned(plugin, old_pid):
+    """Wait for a live successor.  Observed under the plugin's own lock,
+    so a respawn in progress is never seen half-started."""
+
+    while True:
+        async with plugin._stream_lock:
+            proc = plugin._stream_broker._proc
+            if proc is not None and proc.pid != old_pid and proc.is_alive():
+                return
+        await asyncio.sleep(0.05)
+
+
+async def test_shutdown_stops_the_broker_and_the_supervisor(plugin):
+    await plugin.stream_addresses()
+    broker, supervisor = plugin._stream_broker, plugin._supervisor
+
+    await plugin.shutdown()
+
+    assert not broker.is_alive()
+    assert supervisor.done()
+    assert plugin._stream_broker is None

From df7b208f5162781c226ca24ea4e584ca9ac7533a Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:47:30 +0200
Subject: [PATCH 17/19] Check in the in-situ latency harness and document the
 service

`perf/bench_insitu.py` is the section 6 benchmark, parameterized so it
runs against the integration stack (`--broker` / `--endpoint`, or the
usual ORBIT resolution).  Re-measured here: ~11 ms p50 in-process versus
~19 ms p50 through an ORBIT endpoint with the notify window at 0, and
~334 vs ~35 tasks/s under 50-way concurrency -- the placement decision
holds.

The README gains a service section: how to host the plugin
(`--plugins default,dt` plus a rhapsody endpoint), a client snippet, the
two contract notes users trip over (task return values must be JSON-safe
or bytes; persistent bodies are not function tasks), and the R7 binding
policy -- the embedded stream broker is loopback-by-default and a
non-loopback bind needs an explicit decision and a firewalled network.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 README.md            |  88 +++++++++++++++++++++++++++-
 perf/README.md       |  35 +++++++++++
 perf/bench_insitu.py | 134 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 256 insertions(+), 1 deletion(-)
 create mode 100644 perf/README.md
 create mode 100644 perf/bench_insitu.py

diff --git a/README.md b/README.md
index 6b2fcef..833d72f 100644
--- a/README.md
+++ b/README.md
@@ -25,10 +25,12 @@ Not yet implemented:
 
 ## Running the unit tests:
 
-1. `pip install .[test]`
+1. `pip install .[test,service]`
 2. `pytest` (or `tox` for all supported interpreters)
 
 The unit tests start their own stream broker on a random port; no setup.
+The integration tests under `test/integration` bring up a real ORBIT
+broker and rhapsody endpoint and skip themselves when they cannot.
 
 ## Running the demos:
 
@@ -65,3 +67,87 @@ in every subscriber.  A non-loopback bind needs an explicit configuration
 and a private/firewalled network.  External channels are decoded with the
 codec their binding names: `json` (the default) and `raw` are safe to
 accept from a producer you do not control, `cloudpickle` is not.
+
+## Running it as a service (the `dt` ORBIT plugin)
+
+`digitaltwin.service` exposes the framework as a long-running ORBIT
+plugin: one session per client, many independent twins per session, and
+twins that keep running while their client is away.  Installing the
+package registers the plugin through the `radical.orbit.plugins` entry
+point, so a broker or endpoint only has to be told to host it.
+
+```sh
+pip install .[service]
+
+# 1 - the broker, hosting the dt plugin
+radical-orbit-broker.py --plugins default,dt
+
+# 2 - a rhapsody endpoint: where the twins' tasks execute.  The notify
+#     window costs 250 ms on every sequential prediction at its default
+radical-orbit-endpoint.py -n dt_task_ep
+#     ... started with:
+#     RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0
+#     RADICAL_ORBIT_RHAPSODY_BACKEND=concurrent
+```
+
+The client:
+
+```python
+from radical.orbit import EndpointRuntime
+from digitaltwin.components import NULL_DTYPE, TRUTHY
+from digitaltwin.service import register_user_modules
+
+import my_components                       # not installed on the service
+register_user_modules([my_components])
+
+rt = EndpointRuntime()
+rt.start(wait=True)
+
+# 'broker' is the participant hosting dt; engine wiring is explicit
+dt = rt.get_plugin('broker', 'dt', config={
+    'engines': {'task': {'endpoint_name': 'dt_task_ep',
+                         'backends': ['concurrent']}}})
+
+twin = dt.create_twin()                    # polls until the twin is ready
+dt.add_task(twin, dt.package(MySensor), TRUTHY, SENSOR, is_persistent=True)
+dt.add_investigator(twin, dt.package(MyModel), SENSOR, PREDICTION)
+dt.start(twin)
+
+print(dt.twin_list())                      # the observation mechanism
+answer = dt.get_inference(twin, TypedData(SENSOR, 5), PREDICTION)
+
+dt.twin_close(twin)
+```
+
+The session outlives the client: reattach with
+`rt.get_plugin('broker', 'dt', sid=<sid>)` and the twins are still
+there.  `dt.admin_sessions()` lists every session, twin, state and last
+error on the service -- which is how orphans are found and torn down.
+`test/09-service/` is a complete worked example.
+
+Two contract notes:
+
+- A task's *arguments* are cloudpickled, but its **return value must be
+  JSON-safe or `bytes`** -- ORBIT's rhapsody plugin JSON-encodes results
+  and stringifies anything else.  Return plain values from
+  `@flow.function_task` bodies and wrap them in `TypedData` in the
+  component.
+- Persistent components run inline on the service's event loop.  Their
+  bodies must be thin async glue publishing through
+  `runtime.stream`, never `@flow.function_task`s (the service warns when
+  it sees one).
+
+### Binding policy for the service (R7)
+
+The plugin runs its own DT stream broker, embedded, one per plugin and
+shared by every twin.  **It binds to loopback on a random port by
+default, and that default is the safe one**: its payloads are
+cloudpickled, so anyone who can reach the XSUB/XPUB ports gets code
+execution in every subscriber -- weaker than the token-authenticated
+ORBIT channel around it.
+
+A non-loopback bind is possible (`DT_STREAM_PUB_ADDR` /
+`DT_STREAM_SUB_ADDR` on the service host) but requires a deliberate
+decision *and* a firewalled or private network.  Until the data plane
+moves inside ORBIT's authenticated channel, do not expose those ports --
+including in demos.
diff --git a/perf/README.md b/perf/README.md
new file mode 100644
index 0000000..3469e77
--- /dev/null
+++ b/perf/README.md
@@ -0,0 +1,35 @@
+# Performance harnesses
+
+## `bench_insitu.py` — in-situ inference latency
+
+The harness behind the compute-placement decision: one no-op asyncflow
+function task per call, awaited sequentially (the shape of an in-situ
+prediction), plus a 50-way concurrency probe.
+
+```sh
+# against the integration stack (test/integration/conftest.py starts one)
+python perf/bench_insitu.py both \
+    --broker https://127.0.0.1:8031 --endpoint dt_test_task_ep
+```
+
+Three configurations matter, and only one of them is a client-side knob:
+
+| path                                            | measured (loopback) |
+|-------------------------------------------------|---------------------|
+| in-process `ProcessPoolExecutor`                 | ~11 ms p50, ~35 tasks/s concurrent |
+| orbit, `batch_window=0`, notify window 0.25 s    | ~260 ms p50 |
+| orbit, `batch_window=0`, notify window 0         | ~19 ms p50, ~334 tasks/s concurrent |
+
+`batch_window=0` is hardcoded in the harness (and in the service's
+engines).  The **notify window is an endpoint setting** — start the
+endpoint with `RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0` for the fast row
+and leave it at its 0.25 s default to reproduce the slow one.
+
+Routing all user compute through the Rhapsody abstraction therefore
+costs single-digit milliseconds per sequential prediction and wins by an
+order of magnitude under concurrency.
+
+## `streaming_learner_perf.py`, `plot_streaming_perf.py`
+
+Throughput of the streaming active learner (ROSE); unrelated to the
+service path.
diff --git a/perf/bench_insitu.py b/perf/bench_insitu.py
new file mode 100644
index 0000000..5390229
--- /dev/null
+++ b/perf/bench_insitu.py
@@ -0,0 +1,134 @@
+"""In-situ latency benchmark: local process pool vs ORBIT endpoint.
+
+Models the DT in-situ inference path -- one prediction per call, awaited
+sequentially -- plus a concurrency probe.  This is the harness behind
+the compute-placement decision in the DTaaS plan (section 6): routing
+all user compute through the Rhapsody abstraction costs single-digit
+milliseconds per sequential prediction and wins under concurrency.
+
+Run it against the integration stack (see `test/integration/conftest.py`
+for how that stack is started, or start one by hand)::
+
+    python perf/bench_insitu.py both \\
+        --broker https://127.0.0.1:8031 --endpoint dt_test_task_ep
+
+The server-side notification window is an *endpoint* setting and is the
+dominating term when it is not zero -- start the endpoint with
+`RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0` for the fast row, and leave it
+at its 0.25 s default to reproduce the slow one.
+"""
+
+import argparse
+import asyncio
+import os
+import statistics
+import time
+
+from concurrent.futures import ProcessPoolExecutor
+
+from radical.asyncflow import WorkflowEngine  # type: ignore
+
+N_WARM = 20
+N_MEAS = 200
+N_CONCURRENT = 50
+
+
+def report(name: str, latencies: list) -> None:
+    values = sorted(x * 1000 for x in latencies)
+    n = len(values)
+
+    def pct(q):
+        return values[min(n - 1, int(q * n))]
+
+    print(
+        f"{name:28s} n={n}  mean={statistics.mean(values):7.2f}ms  "
+        f"p50={pct(0.50):7.2f}  p90={pct(0.90):7.2f}  p99={pct(0.99):7.2f}  "
+        f"max={values[-1]:7.2f}"
+    )
+
+
+async def measure(engine, label: str, n_meas: int, n_conc: int) -> None:
+    @engine.function_task
+    async def nop():
+        return 42
+
+    for _ in range(N_WARM):
+        await nop()
+
+    latencies = []
+    for _ in range(n_meas):
+        start = time.perf_counter()
+        answer = await nop()
+        latencies.append(time.perf_counter() - start)
+        assert answer == 42, answer
+
+    report(label, latencies)
+
+    start = time.perf_counter()
+    await asyncio.gather(*[nop() for _ in range(n_conc)])
+    elapsed = time.perf_counter() - start
+
+    print(f"{label:28s} {n_conc} concurrent tasks in {elapsed * 1000:.1f}ms "
+          f"({n_conc / elapsed:.0f} tasks/s)")
+
+
+async def bench_local(args) -> None:
+    from rhapsody.backends.execution.concurrent import (  # type: ignore
+        ConcurrentExecutionBackend,
+    )
+
+    backend = await ConcurrentExecutionBackend(ProcessPoolExecutor())
+    engine = await WorkflowEngine.create(backend=backend)
+    try:
+        await measure(engine, "local concurrent pool", args.tasks,
+                      args.concurrent)
+    finally:
+        await engine.shutdown()
+
+
+async def bench_orbit(args) -> None:
+    from rhapsody.backends.execution.orbit import (  # type: ignore
+        OrbitExecutionBackend,
+    )
+
+    backend = await OrbitExecutionBackend(
+        broker_url=args.broker,
+        endpoint_name=args.endpoint,
+        backends=args.backends,
+        batch_window=0,  # no client-side batching: per-call latency
+    )
+    engine = await WorkflowEngine.create(backend=backend)
+    try:
+        await measure(engine, f"orbit {args.endpoint or '<auto>'}",
+                      args.tasks, args.concurrent)
+    finally:
+        await engine.shutdown()
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("which", nargs="?", default="both",
+                        choices=["local", "orbit", "both"])
+    parser.add_argument("--broker", default=os.environ.get(
+        "RADICAL_ORBIT_BROKER_URL"),
+        help="ORBIT broker URL (default: ORBIT's own resolution)")
+    parser.add_argument("--endpoint", default=os.environ.get(
+        "DT_TASK_ENDPOINT"),
+        help="endpoint to run the tasks on (default: auto-select)")
+    parser.add_argument("--backends", default="concurrent",
+                        help="comma-separated remote backends")
+    parser.add_argument("--tasks", type=int, default=N_MEAS)
+    parser.add_argument("--concurrent", type=int, default=N_CONCURRENT)
+
+    args = parser.parse_args()
+    args.backends = [b for b in args.backends.split(",") if b]
+
+    if args.which in ("local", "both"):
+        await bench_local(args)
+
+    if args.which in ("orbit", "both"):
+        await bench_orbit(args)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

From e7a7648a11a1d27f8d95e613490675288375dba3 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 16:51:41 +0200
Subject: [PATCH 18/19] Keep the service tests skippable, and propagate real
 cancellations

- `test/unit/test_service.py` importorskips ORBIT and the integration
  conftest sets `collect_ignore_glob` when it is missing, so an install
  without the `service` extra still runs the framework suite.
- `twin_call` only converts a `CancelledError` into "twin was closed"
  when the handler itself is not being cancelled -- a host shutdown must
  keep propagating.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/digitaltwin/service/session.py |  6 +++++-
 test/integration/conftest.py       |  9 +++++----
 test/unit/test_service.py          | 14 ++++++++------
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/digitaltwin/service/session.py b/src/digitaltwin/service/session.py
index baa349a..f2b2e9b 100644
--- a/src/digitaltwin/service/session.py
+++ b/src/digitaltwin/service/session.py
@@ -227,7 +227,11 @@ async def twin_call(
         except HTTPException:
             raise
         except asyncio.CancelledError:
-            # the twin was closed under an in-flight call
+            # the twin was closed under an in-flight call -- but a
+            # cancellation aimed at *this* handler (host shutdown) has to
+            # keep propagating
+            if asyncio.current_task().cancelling():
+                raise
             raise HTTPException(
                 status_code=409, detail=f"twin {twin_id} was closed during {verb}"
             ) from None
diff --git a/test/integration/conftest.py b/test/integration/conftest.py
index 1c0d6ff..83c20c9 100644
--- a/test/integration/conftest.py
+++ b/test/integration/conftest.py
@@ -24,11 +24,12 @@
 
 import pytest
 
-pytest.importorskip("radical.orbit")
+try:
+    import httpx
 
-import httpx  # noqa: E402  (after the importorskip)
-
-from radical.orbit import EndpointRuntime  # noqa: E402
+    from radical.orbit import EndpointRuntime
+except ImportError:  # no ORBIT installed: there is nothing here to run
+    collect_ignore_glob = ["*"]
 
 REPO = Path(__file__).resolve().parents[2]
 SRC = REPO / "src"
diff --git a/test/unit/test_service.py b/test/unit/test_service.py
index 70a8fc3..420366d 100644
--- a/test/unit/test_service.py
+++ b/test/unit/test_service.py
@@ -11,13 +11,15 @@
 import cloudpickle
 import pytest
 
-from fastapi import FastAPI
-from starlette.testclient import TestClient
+pytest.importorskip("radical.orbit")
 
-from digitaltwin.components import UtilityTask
-from digitaltwin.service.plugin import PluginDT
-from digitaltwin.service.session import DTSession, TwinInstance
-from digitaltwin.service.wire import (
+from fastapi import FastAPI  # noqa: E402
+from starlette.testclient import TestClient  # noqa: E402
+
+from digitaltwin.components import UtilityTask  # noqa: E402
+from digitaltwin.service.plugin import PluginDT  # noqa: E402
+from digitaltwin.service.session import DTSession, TwinInstance  # noqa: E402
+from digitaltwin.service.wire import (  # noqa: E402
     MAX_PAYLOAD,
     Package,
     check_versions,

From 471cf750587d194e75832d00eae06965f5300806 Mon Sep 17 00:00:00 2001
From: Andre Merzky <andre@merzky.net>
Date: Sat, 15 Aug 2026 18:33:10 +0200
Subject: [PATCH 19/19] Review findings: decode later, bound harder, own the
 engine build

Nine findings from the M1 review, no behaviour the API promises changes.

Wire and validation:
- `twin_call` hands the payload to the session as an opaque blob and
  unpickles it only after `_forward` has established that the sid names
  a live session -- decoding is arbitrary code execution and must not
  happen for an unknown or expired one.
- The version stamp carries `digitaltwin` too, compared exactly:
  a shipped component class pickles the framework by reference, so the
  two sides have to be running the same code, not merely compatible
  pickle machinery.
- A hand-built payload with the wrong arity (`TypeError`) is a 409, and
  a call on a closed session is a 410 -- both were 500s.
- `get_inference` clamps a missing or nonsensical timeout to the
  default, so the service-side wait is literally always bounded.

Engines:
- The build is now a session-owned task that callers `shield`-await.  A
  twin whose initialization is cancelled halfway used to cancel the
  build with it and could strand a live `OrbitExecutionBackend` that
  `close()` never saw; a build landing after the session closed now
  shuts itself down.  `close()` raises the inactive flag first and
  drains outstanding builds briefly rather than waiting out a 150 s
  initialization.

Tests:
- The in-flight `twin_close` case runs twice: once waiting in the
  service, once on a real task on the endpoint, so the best-effort
  backend-cancel path is exercised.  That task's delay is deliberately
  short -- the cancel does not reach the endpoint, so a long one would
  hold a slot for the rest of the suite.
- The fd-leak assertion counts descriptors of *the* broker under test
  (pid from the fixture) and fails rather than skips when it cannot;
  the guard-warning assertion is scoped to its own twin id; `_spawn`'s
  log handles are closed on teardown.
- Unit coverage for all of the above, including engine-build
  cancellation and self-disposal.

Also: the twin-id uniqueness scan and the insertion it guards are
race-free only because nothing awaits between them -- pinned with a
comment at both halves.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 README.md                           |   7 +-
 src/digitaltwin/service/plugin.py   |  24 ++--
 src/digitaltwin/service/session.py  | 147 +++++++++++++++++++++---
 src/digitaltwin/service/wire.py     |  42 +++++--
 test/integration/conftest.py        |  65 ++++++++---
 test/integration/test_dt_service.py |  59 ++++++----
 test/integration/twin_components.py |  32 ++++++
 test/unit/test_service.py           | 168 +++++++++++++++++++++++++---
 8 files changed, 450 insertions(+), 94 deletions(-)

diff --git a/README.md b/README.md
index 833d72f..9ffdc1f 100644
--- a/README.md
+++ b/README.md
@@ -125,8 +125,13 @@ there.  `dt.admin_sessions()` lists every session, twin, state and last
 error on the service -- which is how orphans are found and torn down.
 `test/09-service/` is a complete worked example.
 
-Two contract notes:
+Three contract notes:
 
+- The client and the service must run **the same `digitaltwin` version**
+  (and compatible Python / cloudpickle): shipped component classes
+  pickle the framework by reference.  Every call carries those versions
+  and the service rejects skew with a clear error rather than failing
+  somewhere inside an unpickle.
 - A task's *arguments* are cloudpickled, but its **return value must be
   JSON-safe or `bytes`** -- ORBIT's rhapsody plugin JSON-encodes results
   and stringifies anything else.  Return plain values from
diff --git a/src/digitaltwin/service/plugin.py b/src/digitaltwin/service/plugin.py
index 9e96930..42f8e84 100644
--- a/src/digitaltwin/service/plugin.py
+++ b/src/digitaltwin/service/plugin.py
@@ -34,7 +34,6 @@
 from ..streaming import ZMQ_BrokerProcess
 from .client import DTClient
 from .session import VERBS, DTSession
-from .wire import check_versions, decode
 
 log = logging.getLogger("radical.orbit")
 
@@ -169,6 +168,10 @@ async def twin_create(self, request: Request) -> dict:
         data = await _body(request)
 
         twin_id = _twin_id(data.get("twin_id"))
+
+        # Scan-then-insert is race-free only because nothing awaits
+        # between this check and the insertion in `DTSession.twin_create`
+        # -- keep both halves synchronous with respect to each other.
         owner = self._twin_owner(twin_id)
         if owner not in (None, sid):
             raise http_exception(
@@ -198,7 +201,7 @@ async def twin_call(self, request: Request) -> dict:
 
         Body: `{"verb": str, "payload": <b64 cloudpickle of
         {"args": [...], "kwargs": {...}}>, "client": {"python":…,
-        "cloudpickle":…}}`.
+        "cloudpickle":…, "digitaltwin":…}}`.
         """
 
         sid = request.path_params["sid"]
@@ -211,23 +214,16 @@ async def twin_call(self, request: Request) -> dict:
                 ValueError(f"unknown verb {verb!r}; expected one of {', '.join(VERBS)}")
             )
 
-        try:
-            check_versions(data.get("client"))
-            call = decode(data.get("payload") or "") if data.get("payload") else {}
-        except ValueError as exc:
-            raise http_exception(exc) from exc
-        except Exception as exc:
-            raise http_exception(
-                ValueError(f"undecodable {verb} payload: {exc}")
-            ) from exc
-
+        # The payload stays a blob here: unpickling is arbitrary code
+        # execution, so it happens only after `_forward` has established
+        # that this sid names a live session (404 / 410 otherwise).
         return await self._forward(
             sid,
             DTSession.twin_call,
             twin_id=twin_id,
             verb=verb,
-            args=tuple(call.get("args") or ()),
-            kwargs=dict(call.get("kwargs") or {}),
+            payload=data.get("payload"),
+            stamp=data.get("client"),
         )
 
     async def admin_sessions(self, request: Request) -> dict:
diff --git a/src/digitaltwin/service/session.py b/src/digitaltwin/service/session.py
index f2b2e9b..dc186f3 100644
--- a/src/digitaltwin/service/session.py
+++ b/src/digitaltwin/service/session.py
@@ -21,7 +21,7 @@
 from ..components import DataType, TypedData
 from ..runtime import DTRuntime
 from ..streaming import PubSubClient, connect_stream_client
-from .wire import Package, encode
+from .wire import Package, check_versions, decode, encode
 
 log = logging.getLogger("radical.orbit")
 
@@ -38,6 +38,7 @@
 STREAM_CONNECT_TIMEOUT = 30.0
 TWIN_STOP_TIMEOUT = 10.0
 ENGINE_SHUTDOWN_TIMEOUT = 30.0
+ENGINE_BUILD_DRAIN_TIMEOUT = 5.0
 
 # get_inference is the only potentially-long verb; the client may pick its
 # own bound.  [P0-interim] large but finite -- see the plan's section 3.
@@ -59,6 +60,17 @@
 )
 
 
+def _retrieve_exception(task: asyncio.Task) -> None:
+    """Consume a background task's exception.
+
+    A build whose only caller was cancelled is nobody's to await, and an
+    unretrieved exception surfaces as loop noise at teardown.
+    """
+
+    if not task.cancelled():
+        task.exception()
+
+
 class TwinInstance:
     """One twin: a `DTRuntime`, its stream client, and a state machine.
 
@@ -165,6 +177,9 @@ def __init__(self, sid: str, config: Optional[dict] = None):
 
         self.twins: dict[str, TwinInstance] = {}
         self._engines: dict[str, WorkflowEngine] = {}
+        # in-flight builds, owned by the session rather than by whichever
+        # twin asked first (see `engine`)
+        self._engine_tasks: dict[str, asyncio.Task] = {}
         self._engine_lock = asyncio.Lock()
 
     # -- twin lifecycle -----------------------------------------------------
@@ -180,6 +195,9 @@ async def twin_create(self, twin_id: str, config: Optional[dict] = None) -> dict
 
         self._check_active()
 
+        # The plugin checked plugin-wide twin-id uniqueness just before
+        # calling us; that check and this insertion are only race-free
+        # because there is no await between them.  Keep it that way.
         twin = self.twins.get(twin_id)
         if twin is None:
             twin = TwinInstance(twin_id, config)
@@ -212,17 +230,24 @@ async def twin_close(self, twin_id: str) -> dict:
         return self._twin_state(twin)
 
     async def twin_call(
-        self, twin_id: str, verb: str, args: tuple = (), kwargs: Optional[dict] = None
+        self, twin_id: str, verb: str, payload: Optional[str] = None,
+        stamp: Optional[dict] = None
     ) -> dict:
-        """Apply exactly one graph verb to one twin."""
+        """Apply exactly one graph verb to one twin.
+
+        The payload is unpickled here rather than in the route handler:
+        decoding is arbitrary code execution, so it happens only once the
+        session is known to exist and to be live.
+        """
 
         self._check_active()
 
+        args, kwargs = self._decode_call(verb, payload, stamp)
         twin = self._live_twin(twin_id)
         handler = getattr(self, f"_verb_{verb}")
 
         try:
-            extra = await handler(twin, *args, **(kwargs or {}))
+            extra = await handler(twin, *args, **kwargs)
 
         except HTTPException:
             raise
@@ -239,10 +264,11 @@ async def twin_call(
             raise HTTPException(
                 status_code=504, detail=f"twin {twin_id}: {verb} timed out"
             ) from None
-        except (RuntimeError, ValueError, AssertionError) as exc:
+        except (RuntimeError, ValueError, AssertionError, TypeError) as exc:
             # the runtime's own refusals (stopped graph, start-after-stop,
-            # bad dtypes) are client errors, not service faults -- but the
-            # traceback still belongs in the service log
+            # bad dtypes) and a malformed call (wrong arity from a
+            # hand-built payload) are client errors, not service faults --
+            # but the traceback still belongs in the service log
             log.warning("[dt] twin %s: %s failed", twin_id, verb, exc_info=exc)
             raise HTTPException(
                 status_code=409,
@@ -251,6 +277,40 @@ async def twin_call(
 
         return {**self._twin_state(twin), **(extra or {})}
 
+    def _decode_call(self, verb: str, payload: Optional[str],
+                     stamp: Optional[dict]) -> tuple:
+        """Version-check and unpickle one verb's arguments."""
+
+        try:
+            check_versions(stamp)
+            call = decode(payload) if payload else {}
+
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except Exception as exc:
+            raise HTTPException(
+                status_code=400,
+                detail=f"undecodable {verb} payload: {type(exc).__name__}: {exc}",
+            ) from exc
+
+        if not isinstance(call, dict):
+            raise HTTPException(
+                status_code=400, detail=f"malformed {verb} payload")
+
+        return tuple(call.get("args") or ()), dict(call.get("kwargs") or {})
+
+    def _check_active(self) -> None:
+        """A call on a closed session is `410 Gone`, not a service fault.
+
+        The base raises `RuntimeError`, which `Plugin._forward` would turn
+        into a 500 -- but a client racing its own `unregister_session` has
+        simply outlived its session.
+        """
+
+        if not self.is_active:
+            raise HTTPException(
+                status_code=410, detail=f"session {self.sid} is closed")
+
     # -- verbs --------------------------------------------------------------
 
     async def _verb_add_task(
@@ -308,6 +368,11 @@ async def _verb_get_inference(
         output_dtype: DataType,
         timeout: Optional[float] = DEFAULT_INFERENCE_TIMEOUT,
     ) -> dict:
+        # the service-side wait is always bounded: a client asking for
+        # `None` (or nonsense) gets the default, not an unbounded hold
+        if not isinstance(timeout, (int, float)) or timeout <= 0:
+            timeout = DEFAULT_INFERENCE_TIMEOUT
+
         # run it as a tracked task so `twin_close` can cancel it -- an
         # in-flight inference must not outlive (or hold up) its twin
         task = twin.track(
@@ -323,13 +388,54 @@ async def engine(self, name: str = TASK_ENGINE) -> WorkflowEngine:
 
         One engine per name per session, never per twin.  M1 only ever
         asks for `'task'`.
+
+        The build is a *session-owned* task that callers only ever
+        `shield`-await: a twin whose initialization is cancelled halfway
+        through must not take the build down with it and strand a live
+        `OrbitExecutionBackend` that nothing holds a reference to.  A
+        build that lands after the session closed disposes of itself.
         """
 
         async with self._engine_lock:
             flow = self._engines.get(name)
-            if flow is None:
-                flow = self._engines[name] = await self._create_engine(name)
-            return flow
+            if flow is not None:
+                return flow
+
+            task = self._engine_tasks.get(name)
+            if task is None or task.done():  # first build, or a retry
+                task = self._engine_tasks[name] = asyncio.ensure_future(
+                    self._build_engine(name)
+                )
+                task.add_done_callback(_retrieve_exception)
+
+        return await asyncio.shield(task)
+
+    async def _build_engine(self, name: str) -> WorkflowEngine:
+        flow = await self._create_engine(name)
+
+        if not self.is_active:
+            # every twin that wanted this engine is gone; nothing will
+            # ever own it, so it must not outlive its own construction
+            await self._shutdown_engine(name, flow)
+            raise RuntimeError(
+                f"session {self.sid} closed while engine {name!r} was building")
+
+        self._engines[name] = flow
+
+        return flow
+
+    async def _shutdown_engine(self, name: str, flow: WorkflowEngine) -> None:
+        """Bounded engine teardown.
+
+        asyncflow's own shutdown is an unbounded gather, so a bare await
+        could park the host loop.
+        """
+
+        try:
+            await asyncio.wait_for(flow.shutdown(), ENGINE_SHUTDOWN_TIMEOUT)
+        except Exception as exc:
+            log.warning("[dt] session %s: engine %r shutdown: %s",
+                        self.sid, name, exc)
 
     async def _create_engine(self, name: str) -> WorkflowEngine:
         cfg = (self.config.get("engines") or {}).get(name) or {}
@@ -361,20 +467,27 @@ def broker_url(self) -> Optional[str]:
     async def close(self) -> dict:
         """Stop every twin, then shut the engines down.
 
-        Engine shutdown is `wait_for`-bounded: asyncflow's own shutdown is
-        an unbounded gather, so a bare await could park the host loop.
+        The inactive flag goes up first: an engine build still in flight
+        reads it when it lands and disposes of itself, so teardown never
+        has to wait out a 150 s backend initialization to avoid leaking
+        it.
         """
 
+        self._active = False
+
         for twin in list(self.twins.values()):
             await twin.close()
         self.twins.clear()
 
+        # give a build that is nearly done a moment to land and self-
+        # dispose; anything slower cleans up after us, unsupervised
+        builds = [t for t in self._engine_tasks.values() if not t.done()]
+        if builds:
+            await asyncio.wait(builds, timeout=ENGINE_BUILD_DRAIN_TIMEOUT)
+        self._engine_tasks.clear()
+
         for name, flow in self._engines.items():
-            try:
-                await asyncio.wait_for(flow.shutdown(), ENGINE_SHUTDOWN_TIMEOUT)
-            except Exception as exc:
-                log.warning("[dt] session %s: engine %r shutdown: %s",
-                            self.sid, name, exc)
+            await self._shutdown_engine(name, flow)
         self._engines.clear()
 
         return await super().close()
diff --git a/src/digitaltwin/service/wire.py b/src/digitaltwin/service/wire.py
index 440b284..227d2bd 100644
--- a/src/digitaltwin/service/wire.py
+++ b/src/digitaltwin/service/wire.py
@@ -17,6 +17,7 @@
 import sys
 
 from dataclasses import dataclass, field
+from importlib import metadata
 from typing import Any
 
 import cloudpickle
@@ -60,44 +61,61 @@ def register_user_modules(modules: list) -> None:
         cloudpickle.register_pickle_by_value(module)
 
 
+def _dt_version() -> str:
+    try:
+        return metadata.version("digitaltwin")
+    except metadata.PackageNotFoundError:  # running from a source tree
+        return "unknown"
+
+
 def version_stamp() -> dict:
-    """The interpreter/cloudpickle identity a pickle was produced with."""
+    """The identity a pickle was produced with.
+
+    `digitaltwin` is in here because a shipped component class pickles
+    its framework base classes, dtypes and `Package` *by reference*: the
+    two sides have to be looking at the same code, not merely at
+    compatible pickle machinery.
+    """
 
     return {
         "python": "%d.%d" % sys.version_info[:2],
         "cloudpickle": cloudpickle.__version__,
+        "digitaltwin": _dt_version(),
     }
 
 
+def _minor(version: str) -> tuple:
+    return tuple(str(version).split(".")[:2])
+
+
+# How closely each version has to line up.  Python and cloudpickle break
+# pickle compatibility at minor-version granularity; `digitaltwin` is
+# resolved by reference, so any difference at all is a different service.
+_COMPARE = {"python": _minor, "cloudpickle": _minor, "digitaltwin": str}
+
+
 def check_versions(stamp: dict) -> None:
     """Reject a payload produced by a skewed client.
 
     Raises:
-        ValueError: on a Python minor-version or cloudpickle
-            minor-version mismatch, naming both sides.
+        ValueError: on a missing or mismatched version, naming both sides.
     """
 
-    ours = version_stamp()
     theirs = stamp or {}
 
-    for key, mine in ours.items():
+    for key, mine in version_stamp().items():
         yours = theirs.get(key)
         if yours is None:
             raise ValueError(f"client did not report its {key} version")
 
-        # compare on major.minor: that is the granularity at which
-        # pickle compatibility actually breaks
-        if _minor(yours) != _minor(mine):
+        compare = _COMPARE[key]
+        if compare(yours) != compare(mine):
             raise ValueError(
                 f"{key} version skew: client {yours}, service {mine}"
                 f" -- cloudpickled payloads are not portable across those"
             )
 
 
-def _minor(version: str) -> tuple:
-    return tuple(str(version).split(".")[:2])
-
-
 def encode(obj: Any) -> str:
     """Cloudpickle `obj` into a JSON-safe string."""
 
diff --git a/test/integration/conftest.py b/test/integration/conftest.py
index 83c20c9..6c43465 100644
--- a/test/integration/conftest.py
+++ b/test/integration/conftest.py
@@ -21,6 +21,7 @@
 import uuid
 
 from pathlib import Path
+from typing import NamedTuple
 
 import pytest
 
@@ -76,24 +77,42 @@ def _child_env(**extra: str) -> dict:
 
 
 def _spawn(name: str, argv: list, **env: str) -> subprocess.Popen:
+    """Start a child, logging to `LOGS/<name>.log`.
+
+    The log handle rides on the process object so `_terminate` can close
+    it -- these fixtures are session-scoped, but the pytest process must
+    not accumulate descriptors either.
+    """
+
     LOGS.mkdir(parents=True, exist_ok=True)
     log = (LOGS / f"{name}.log").open("w")
 
-    return subprocess.Popen(
-        argv, env=_child_env(**env), stdout=log, stderr=subprocess.STDOUT
-    )
+    try:
+        proc = subprocess.Popen(
+            argv, env=_child_env(**env), stdout=log, stderr=subprocess.STDOUT
+        )
+    except BaseException:
+        log.close()
+        raise
 
+    proc._dt_log = log
 
-def _terminate(proc: subprocess.Popen, timeout: float = 15.0) -> None:
-    if proc.poll() is not None:
-        return
+    return proc
 
-    proc.terminate()
+
+def _terminate(proc: subprocess.Popen, timeout: float = 15.0) -> None:
     try:
-        proc.wait(timeout)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-        proc.wait(5)
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait(5)
+    finally:
+        log = getattr(proc, "_dt_log", None)
+        if log is not None and not log.closed:
+            log.close()
 
 
 def _orbit_script(name: str) -> list:
@@ -117,6 +136,15 @@ def _orbit_script(name: str) -> list:
     pytest.skip(f"cannot locate {name}")
 
 
+class Broker(NamedTuple):
+    """The broker under test: where to reach it, and which process it is
+    (tests assert against *this* broker's resources, never against
+    whatever else on the host happens to look like a broker)."""
+
+    url: str
+    pid: int
+
+
 @pytest.fixture(scope="session")
 def broker():
     """An ORBIT broker on a non-default loopback port, hosting `dt`."""
@@ -133,7 +161,7 @@ def broker():
 
     try:
         _await_broker(proc)
-        yield BROKER_URL
+        yield Broker(BROKER_URL, proc.pid)
     finally:
         _terminate(proc)
 
@@ -159,7 +187,7 @@ def task_endpoint(broker):
     """A co-located rhapsody endpoint: where the twins' tasks execute."""
 
     argv = _orbit_script("radical-orbit-endpoint.py") + [
-        "-n", TASK_ENDPOINT, "-u", broker, "-p", "default",
+        "-n", TASK_ENDPOINT, "-u", broker.url, "-p", "default",
     ]
     proc = _spawn(
         TASK_ENDPOINT,
@@ -184,7 +212,7 @@ def dt_endpoint(broker):
     """
 
     argv = _orbit_script("radical-orbit-endpoint.py") + [
-        "-n", DT_ENDPOINT, "-u", broker, "-p", "dt",
+        "-n", DT_ENDPOINT, "-u", broker.url, "-p", "dt",
     ]
     proc = _spawn(DT_ENDPOINT, argv)
 
@@ -224,7 +252,14 @@ def _await_plugin(endpoint: str, plugin: str, proc: subprocess.Popen) -> None:
 def stack(broker, task_endpoint):
     """The full broker + endpoint stack; returns the broker URL."""
 
-    return broker
+    return broker.url
+
+
+@pytest.fixture(scope="session")
+def broker_pid(broker):
+    """The pid of the broker under test -- for resource assertions."""
+
+    return broker.pid
 
 
 @pytest.fixture
diff --git a/test/integration/test_dt_service.py b/test/integration/test_dt_service.py
index 67958b4..afc70d3 100644
--- a/test/integration/test_dt_service.py
+++ b/test/integration/test_dt_service.py
@@ -13,6 +13,8 @@
 import time
 import uuid
 
+from pathlib import Path
+
 import pytest
 
 from radical.orbit import EndpointRuntime
@@ -34,6 +36,7 @@
     MisplacedFunctionTask,
     OffsetModel,
     SlowModel,
+    SlowTaskModel,
 )
 
 pytestmark = pytest.mark.integration
@@ -100,18 +103,22 @@ async def collect(dt, twin, dtype, count, timeout=POLL_TIMEOUT):
         await client.close()
 
 
-def broker_fds():
-    """Open file descriptors of the broker process (Linux only)."""
+def broker_fds(pid):
+    """Open file descriptors of *the* broker under test (Linux only).
 
-    pids = [p for p in os.listdir("/proc") if p.isdigit()]
-    for pid in pids:
-        try:
-            cmdline = open(f"/proc/{pid}/cmdline", "rb").read().decode()
-            if "radical-orbit-broker" in cmdline:
-                return len(os.listdir(f"/proc/{pid}/fd"))
-        except OSError:
-            continue
-    return None
+    The pid comes from the fixture: counting descriptors of whatever
+    process on the host happens to look like a broker would make the
+    leak assertion meaningless.
+    """
+
+    fds = Path(f"/proc/{pid}/fd")
+    if not Path("/proc/self/fd").exists():
+        pytest.skip("no /proc: cannot count descriptors")
+
+    try:
+        return len(os.listdir(fds))
+    except OSError as exc:
+        pytest.fail(f"cannot read {fds} of the broker under test: {exc}")
 
 
 # ---------------------------------------------------------------------------
@@ -175,7 +182,7 @@ def in_band(values, base):
 # lifecycle
 # ---------------------------------------------------------------------------
 
-def test_twin_churn_leaks_nothing(dt):
+def test_twin_churn_leaks_nothing(dt, broker_pid):
     """Create/close cycles must not accumulate twins, engines or fds."""
 
     def cycle():
@@ -187,7 +194,7 @@ def cycle():
         dt.twin_close(twin)
 
     cycle()  # first cycle pays the engine + stream broker startup
-    before = broker_fds()
+    before = broker_fds(broker_pid)
 
     for _ in range(4):
         cycle()
@@ -200,17 +207,22 @@ def cycle():
     assert session["engines"] == ["task"]
     assert session["twins"] == []
 
-    after = broker_fds()
-    if before is not None and after is not None:
-        assert after <= before + 8, f"broker fds {before} -> {after}"
+    after = broker_fds(broker_pid)
+    assert after <= before + 8, f"broker fds {before} -> {after}"
 
 
-def test_twin_close_with_inference_in_flight(dt, twin_id):
+@pytest.mark.parametrize("model", [SlowModel, SlowTaskModel],
+                         ids=["local-wait", "backend-task"])
+def test_twin_close_with_inference_in_flight(dt, twin_id, model):
     """A `twin_close` must not be held up by -- nor strand -- a call in
-    flight; the caller gets a prompt, clear error."""
+    flight; the caller gets a prompt, clear error.
+
+    Twice over: once where the inference waits in the service, and once
+    where it waits on a real task running on the endpoint.
+    """
 
     dt.create_twin(twin_id)
-    dt.add_investigator(twin_id, dt.package(SlowModel), SENSOR_DTYPE,
+    dt.add_investigator(twin_id, dt.package(model), SENSOR_DTYPE,
                         INFERENCE_DTYPE)
     dt.start(twin_id)
 
@@ -388,8 +400,13 @@ def test_persistent_function_task_warns(dt, twin_id):
     dt.add_task(twin_id, dt.package(MisplacedFunctionTask), TRUTHY,
                 NULL_DTYPE, is_persistent=True)
 
-    log = (LOGS / "broker.log").read_text()
-    assert "MisplacedFunctionTask registered 1 function_task" in log
+    # scoped to this twin: the log accumulates across the whole session,
+    # so an unscoped match would pass on a previous test's warning
+    warnings = [line for line in (LOGS / "broker.log").read_text().splitlines()
+                if twin_id in line and "function_task" in line]
+
+    assert len(warnings) == 1, warnings
+    assert "MisplacedFunctionTask registered 1 function_task" in warnings[0]
 
 
 def test_version_skew_is_rejected(dt, twin_id):
diff --git a/test/integration/twin_components.py b/test/integration/twin_components.py
index 630876c..2276aa4 100644
--- a/test/integration/twin_components.py
+++ b/test/integration/twin_components.py
@@ -81,6 +81,38 @@ async def infer(in_data: TypedData, **kwargs):
         runtime.publish_new_model({})
 
 
+class SlowTaskModel(ModelInvestigator):
+    """`SlowModel`, but the wait happens in a real backend task.
+
+    Closing a twin mid-inference then has to walk the best-effort cancel
+    of an in-flight *engine* call, not just of a local sleep.
+
+    The delay is deliberately short: cancelling the local await does not
+    reach into the endpoint (`stop` is best-effort by design), so the
+    task really does keep running there, and a long one would hold a
+    slot on the shared test endpoint for the rest of the suite.
+    """
+
+    def __init__(self, flow, delay: float = 15.0):
+        super().__init__(flow)
+        self.delay = delay
+
+        @flow.function_task
+        async def slow(seconds: float):
+            await asyncio.sleep(seconds)
+            return 1
+
+        self.slow = slow
+
+    async def main_loop(self, runtime):
+        async def infer(in_data: TypedData, **kwargs):
+            await self.slow(self.delay)
+            return TypedData(INFERENCE_DTYPE, in_data.data)
+
+        runtime.set_inference_task(infer)
+        runtime.publish_new_model({})
+
+
 class EchoSink(UtilityTask):
     """Terminal component: republishes what it receives, so a client can
     watch the pipeline over the twin's own (namespaced) stream."""
diff --git a/test/unit/test_service.py b/test/unit/test_service.py
index 420366d..8fb8908 100644
--- a/test/unit/test_service.py
+++ b/test/unit/test_service.py
@@ -6,14 +6,14 @@
 """
 
 import asyncio
-import sys
 
-import cloudpickle
+from typing import Optional
+
 import pytest
 
 pytest.importorskip("radical.orbit")
 
-from fastapi import FastAPI  # noqa: E402
+from fastapi import FastAPI, HTTPException  # noqa: E402
 from starlette.testclient import TestClient  # noqa: E402
 
 from digitaltwin.components import UtilityTask  # noqa: E402
@@ -141,6 +141,50 @@ def test_unknown_session_is_404(client):
     assert client.get("/dt/twin_list/session.nope").status_code == 404
 
 
+def test_an_unknown_session_never_unpickles_the_payload(client):
+    """Decoding is arbitrary code execution: an unroutable verb must be
+    turned away before its payload reaches cloudpickle.
+
+    The payload here is not decodable at all -- a 400 would mean the
+    service tried."""
+
+    resp = client.post(
+        "/dt/twin_call/session.nope/t1",
+        json={"verb": "start", "payload": "!! not base64 !!",
+              "client": version_stamp()},
+    )
+
+    assert resp.status_code == 404
+
+
+async def test_a_call_on_a_closed_session_is_410():
+    """Not a 500: a client racing its own `unregister_session` has
+    simply outlived its session."""
+
+    session = DTSession("s1")
+    await session.close()
+
+    with pytest.raises(HTTPException) as raised:
+        await session.twin_call("t1", "start")
+
+    assert raised.value.status_code == 410
+
+
+async def test_a_malformed_call_is_a_client_error():
+    """A hand-built payload with the wrong arity is a bad request, not a
+    service fault."""
+
+    session = DTSession("s1")
+    session.twins["t1"] = _twin_with(_FakeFlow())
+
+    with pytest.raises(HTTPException) as raised:
+        await session.twin_call("t1", "start",
+                                encode({"args": (1, 2, 3)}), version_stamp())
+
+    assert raised.value.status_code == 409
+    assert "TypeError" in raised.value.detail
+
+
 # ---------------------------------------------------------------------------
 # wire format
 # ---------------------------------------------------------------------------
@@ -164,19 +208,29 @@ def test_version_stamp_matches_itself():
     check_versions(version_stamp())
 
 
-@pytest.mark.parametrize(
-    "stamp",
-    [
-        {},
-        {"python": "2.7", "cloudpickle": cloudpickle.__version__},
-        {"python": "%d.%d" % sys.version_info[:2], "cloudpickle": "0.1"},
-    ],
-)
-def test_version_skew_is_rejected(stamp):
-    with pytest.raises(ValueError):
+@pytest.mark.parametrize("key, value", [
+    ("python", "2.7"),
+    ("cloudpickle", "0.1"),
+    # by-reference pickling of component classes: any difference counts
+    ("digitaltwin", version_stamp()["digitaltwin"] + ".dev1"),
+])
+def test_version_skew_is_rejected(key, value):
+    with pytest.raises(ValueError, match=f"{key} version skew"):
+        check_versions({**version_stamp(), key: value})
+
+
+@pytest.mark.parametrize("missing", ["python", "cloudpickle", "digitaltwin"])
+def test_a_missing_version_is_rejected(missing):
+    stamp = {k: v for k, v in version_stamp().items() if k != missing}
+
+    with pytest.raises(ValueError, match=f"did not report its {missing}"):
         check_versions(stamp)
 
 
+def test_the_stamp_pins_digitaltwin_too():
+    assert "digitaltwin" in version_stamp()
+
+
 def test_package_instantiates_with_the_injected_engine():
     class Component:
         def __init__(self, flow, a, b=0):
@@ -192,15 +246,19 @@ def __init__(self, flow, a, b=0):
 # ---------------------------------------------------------------------------
 
 class _FakeFlow:
-    """Just enough engine for `_instantiate`."""
+    """Just enough engine for `_instantiate` and teardown."""
 
     def __init__(self):
         self.registered = []
+        self.is_shut_down = False
 
     def function_task(self, func):
         self.registered.append(func)
         return func
 
+    async def shutdown(self):
+        self.is_shut_down = True
+
 
 class _Persistent(UtilityTask):
     def __init__(self, flow):
@@ -261,6 +319,88 @@ def test_instantiate_rejects_a_non_package():
         DTSession("s1")._instantiate(_Plain, _twin_with(_FakeFlow()))
 
 
+# ---------------------------------------------------------------------------
+# engines
+# ---------------------------------------------------------------------------
+
+def _slow_build(session, delay: float, started: Optional[asyncio.Event] = None):
+    """Stand in for a real (up to 150 s) backend build."""
+
+    built = []
+
+    async def build(name):
+        if started is not None:
+            started.set()
+        await asyncio.sleep(delay)
+        flow = _FakeFlow()
+        built.append(flow)
+        return flow
+
+    session._create_engine = build
+
+    return built
+
+
+async def test_one_engine_is_shared_by_every_caller():
+    session = DTSession("s1")
+    built = _slow_build(session, 0.05)
+
+    first, second = await asyncio.gather(session.engine(), session.engine())
+
+    assert first is second
+    assert len(built) == 1
+
+
+async def test_an_engine_build_survives_a_cancelled_caller():
+    """A twin whose initialization is cancelled halfway must not take the
+    build with it -- that would strand a live backend nobody holds."""
+
+    session = DTSession("s1")
+    started = asyncio.Event()
+    built = _slow_build(session, 0.2, started)
+
+    caller = asyncio.create_task(session.engine())
+    await started.wait()
+    caller.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await caller
+
+    flow = await session.engine()
+
+    assert built == [flow]
+    assert session._engines == {"task": flow}
+
+
+async def test_an_engine_landing_after_close_disposes_of_itself():
+    """Nothing will ever own it, so it must not outlive its own build."""
+
+    session = DTSession("s1")
+    built = _slow_build(session, 0.2)
+
+    caller = asyncio.create_task(session.engine())
+    await asyncio.sleep(0.05)
+
+    await session.close()
+
+    with pytest.raises((RuntimeError, asyncio.CancelledError)):
+        await caller
+
+    assert len(built) == 1
+    assert built[0].is_shut_down
+    assert session._engines == {}
+
+
+async def test_close_shuts_down_a_built_engine():
+    session = DTSession("s1")
+    _slow_build(session, 0)
+
+    flow = await session.engine()
+    await session.close()
+
+    assert flow.is_shut_down
+    assert session._engines == {}
+
+
 # ---------------------------------------------------------------------------
 # the embedded stream broker and its supervisor
 # ---------------------------------------------------------------------------