Feature/radex stores

dev-resources/radex-config.toml

@@ -4,10 +4,12 @@ addopts = [
 ]
 strict_config = true
 strict_markers = true
+asyncio_mode = "auto"
 markers = [
     "slow: Test may be slow to run",
     "compiled: Test has a component compiled at run time",
     "example: Test is running one of the examples",
+    "redis: Test requires a real redis-server binary on PATH"
 ]
 
 [tool.black]

dev-resources/requirements-dev.txt

@@ -1,3 +1,5 @@
 pytest
+pytest-asyncio
 black
-isort
+isort
+redis

example/py-store-exchange/dragon/driver.py

@@ -0,0 +1,118 @@
+"""
+DragonStore-managed data exchange -- single-file example.
+
+Architecture
+────────────
+  This mirrors example/py-cpp-exchange/dragon/driver.py, but instead of the
+  driver manually constructing/serializing/destroying a `dragon.data.ddict.DDict`
+  itself, RADEX's `DragonStore` owns that lifecycle:
+
+    DragonStore.start()     -> constructs the DDict, blocks until ready
+    DragonStore.endpoints   -> [DragonEndpoint(descriptor=...)]
+    DragonStore.shutdown()  -> destroys the DDict
+
+  DragonStore never constructs a client itself -- a real
+  radex.clients.core.DragonClient is built directly from the endpoint's
+  serialized descriptor, below.
+
+  The serialized descriptor (`store.endpoints[0].serialize()`) is what you'd
+  hand to a separately-launched process (env var, task kwarg, etc.) -- the
+  Store itself never touches os.environ, so that handoff is always explicit.
+
+Run with:
+    dragon -s -- python driver.py
+"""
+
+import asyncio
+import os
+import pathlib
+import time
+
+import numpy as np
+from dragon.native.process import Process, ProcessTemplate
+
+from radex import DragonStore
+from radex.clients.core import DragonClient
+from radex.handles.handles import IncomingHandle, OutgoingHandle
+
+HERE = pathlib.Path(__file__).parent.absolute()
+ROOT = HERE.parent.parent.parent
+EXAMPLES_BIN_DIR = ROOT / "install" / "bin" / "examples"
+
+
+async def main() -> int:
+    # ── 1. RADEX starts and owns the DDict-backed store ─────────────────────
+    store = DragonStore(managers_per_node=1, n_nodes=1)
+    await store.start()
+    print(f"[Driver] DragonStore ready: {store.endpoints[0].serialize()[:32]}...")
+
+    # ── 2. RADEX client, built directly from the endpoint's descriptor --
+    #      Store never constructs clients itself ─────────────────────────
+    client = DragonClient(descriptor=store.endpoints[0].serialize(), timeout=5)
+
+    try:
+        # ── 3. Hand the serialized descriptor to a separately-launched
+        #      process -- the store never sets env vars for you.
+        app_tmpl = ProcessTemplate(
+            target=os.fspath(EXAMPLES_BIN_DIR / "dragon-cpp-with-py"),
+            env={"SERIALIZED_DDICT": store.endpoints[0].serialize()},
+        )
+        app = Process.from_template(app_tmpl)
+
+        print("[Driver] Starting C++ app")
+        app.start()
+        try:
+            time.sleep(3)
+            print("[Driver] Setting Int")
+            client.put_scalar(OutgoingHandle("py-int"), 123)
+
+            time.sleep(3)
+            print("[Driver] Setting Double")
+            client.put_scalar(OutgoingHandle("py-double"), 9.87)
+
+            time.sleep(3)
+            print("[Driver] Setting Numpy Float")
+            client.put_scalar(OutgoingHandle("py-np-float"), np.float32(45.6))
+
+            time.sleep(3)
+            print("[Driver] Setting Int Tensor")
+            client.put_tensor(
+                OutgoingHandle("py-int-tensor"), np.arange(4, dtype=np.int32)
+            )
+
+            time.sleep(3)
+            print("[Driver] Setting Float Tensor")
+            client.put_tensor(
+                OutgoingHandle("py-float-tensor"),
+                np.arange(12, dtype=np.float64).reshape((6, 2)),
+            )
+
+            print("[Driver] Looking for keys")
+            print_scalar(client, "cpp-double")
+            print_scalar(client, "cpp-int")
+            print_tensor(client, "cpp-double-tensor")
+            print_tensor(client, "cpp-long-tensor")
+        finally:
+            app.join()
+    finally:
+        # ── 4. RADEX owns teardown too -- idempotent, safe to call again.
+        await store.shutdown()
+        print(f"[Driver] Store state: {store.state.name}")
+
+    return 0
+
+
+def print_scalar(client, key):
+    print(f"[Driver] Waiting for scalar key `{key}`")
+    scalar = client.wait_for_scalar(IncomingHandle(key), 10)
+    print(f"[Driver] Got scalar: {scalar}")
+
+
+def print_tensor(client, key):
+    print(f"[Driver] Waiting for tensor key `{key}`")
+    tensor = client.wait_for_tensor(IncomingHandle(key), 10)
+    print(f"[Driver] Got tensor: {tensor.ravel()}")
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

example/py-store-exchange/redis/driver.py

@@ -0,0 +1,110 @@
+"""
+RedisStore-managed data exchange -- single-file example.
+
+Architecture
+────────────
+  RedisStore models N *independent* per-node Redis instances -- not a Redis
+  Cluster. Each node is its own isolated keyspace; `store.endpoints` is a
+  list with one RedisEndpoint per node.
+
+  RedisStore never constructs a client itself. The typed RADEX client
+  (`radex.clients.core.RedisClient` -- `put_scalar`/`get_scalar`/
+  `put_tensor`/`get_tensor`, the same API `DragonClient` exposes) only
+  supports env-based construction (no host/port constructor args), so a
+  client for one specific endpoint is built by pointing
+  `RADEX_STORE`/`RADEX_STORE_OPTS` at that endpoint first, then
+  constructing `RedisClient()`:
+
+      os.environ["RADEX_STORE"] = endpoint.serialize()
+      os.environ["RADEX_STORE_OPTS"] = "Standalone"
+      client = RedisClient()
+
+  (For a raw redis-py client instead -- direct SET/GET, not RADEX's typed
+  API -- use `RedisStore.client(index=...)`.)
+
+  Locally, RedisStore() with no arguments spawns a single `redis-server` on
+  an auto-picked free port -- this is what the first half of this example
+  uses. The second half shows the same API scaled out to several
+  independent local nodes, and how you'd point it at an HPC launcher
+  instead (commented out, since it needs a real Slurm allocation to run):
+
+    RedisStore(
+        hosts=["nid00001", "nid00002", "nid00003"],
+        port=6380,
+        cmd="srun --nodelist={host} redis-server --port {port}",
+    )
+
+Run with:
+    python driver.py
+"""
+
+import asyncio
+import os
+
+import numpy as np
+
+from radex import RedisStore
+from radex.clients.core import RedisClient
+from radex.handles.handles import IncomingHandle, OutgoingHandle
+from radex.store.redis_store import RedisEndpoint
+
+
+def client_for(endpoint: RedisEndpoint) -> RedisClient:
+    """Build a typed RADEX client bound to one specific endpoint.
+
+    `RedisClient()` only constructs from the environment, so this points
+    `RADEX_STORE`/`RADEX_STORE_OPTS` at `endpoint` first.
+    """
+    os.environ["RADEX_STORE"] = endpoint.serialize()
+    os.environ["RADEX_STORE_OPTS"] = "Standalone"
+    return RedisClient()
+
+
+async def single_node_demo() -> None:
+    print("── Single local node ──────────────────────────────────────────")
+    store = RedisStore()
+    await store.start()
+    try:
+        endpoint = store.endpoints[0]
+        print(f"[Driver] RedisStore ready at {endpoint.serialize()}")
+
+        client = client_for(endpoint)
+
+        client.put_scalar(OutgoingHandle("greeting-count"), 1)
+        count = client.get_scalar(IncomingHandle("greeting-count"))
+        print(f"[Driver] Got scalar back: {count}")
+
+        client.put_tensor(OutgoingHandle("samples"), np.arange(6, dtype=np.float64))
+        samples = client.get_tensor(IncomingHandle("samples"))
+        print(f"[Driver] Got tensor back: {samples}")
+
+        print(f"[Driver] ready(): {await store.ready()}")
+    finally:
+        await store.shutdown()
+        print(f"[Driver] Store state: {store.state.name}")
+
+
+async def multi_node_demo() -> None:
+    print("\n── Multiple independent local nodes ───────────────────────────")
+    store = RedisStore(hosts=["localhost", "localhost", "localhost"])
+    await store.start()
+    try:
+        print(f"[Driver] {len(store.endpoints)} independent nodes:")
+        for i, endpoint in enumerate(store.endpoints):
+            # Each node is its own keyspace -- write a distinct value to each.
+            client = client_for(endpoint)
+            client.put_scalar(OutgoingHandle("node-id"), i)
+            node_id = client.get_scalar(IncomingHandle("node-id"))
+            print(f"[Driver]   node {i}: {endpoint.serialize()} -> node-id={node_id}")
+    finally:
+        await store.shutdown()
+
+
+async def main() -> int:
+    await single_node_demo()
+    await multi_node_demo()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))

src/python/pyproject.toml

@@ -13,3 +13,7 @@ dependencies = [
     "cloudpickle",
     "numpy",
 ]
+
+[project.optional-dependencies]
+redis = ["redis>=5.0"]
+dragon = []

src/python/src/radex/__init__.py

@@ -0,0 +1,29 @@
+from radex.store import (
+    DragonEndpoint,
+    DragonStore,
+    Endpoint,
+    RedisEndpoint,
+    RedisStore,
+    Store,
+    StoreError,
+    StoreNotReadyError,
+    StoreStartupError,
+    StoreState,
+    StoreStateError,
+    StoreTerminatedError,
+)
+
+__all__ = [
+    "DragonEndpoint",
+    "DragonStore",
+    "Endpoint",
+    "RedisEndpoint",
+    "RedisStore",
+    "Store",
+    "StoreError",
+    "StoreNotReadyError",
+    "StoreStartupError",
+    "StoreState",
+    "StoreStateError",
+    "StoreTerminatedError",
+]

src/python/src/radex/store/__init__.py

@@ -0,0 +1,27 @@
+from radex.store.base import (
+    Endpoint,
+    Store,
+    StoreError,
+    StoreNotReadyError,
+    StoreStartupError,
+    StoreState,
+    StoreStateError,
+    StoreTerminatedError,
+)
+from radex.store.dragon_store import DragonEndpoint, DragonStore
+from radex.store.redis_store import RedisEndpoint, RedisStore
+
+__all__ = [
+    "DragonEndpoint",
+    "DragonStore",
+    "Endpoint",
+    "RedisEndpoint",
+    "RedisStore",
+    "Store",
+    "StoreError",
+    "StoreNotReadyError",
+    "StoreStartupError",
+    "StoreState",
+    "StoreStateError",
+    "StoreTerminatedError",
+]