Begin Running Examples in the Test Suite

.github/workflows/tests.yml

@@ -63,7 +63,7 @@ jobs:
     - name: Install DragonHPC
       run: |
             git clone https://github.com/DragonHPC/dragon.git
-            pushd dragon && git checkout $_DRAGON_VERSION && pushd devtools && source VARIABLES && popd && pip install -e src/
+            pushd dragon && git checkout $_DRAGON_VERSION && pushd devtools && source VARIABLES && popd && pip install src/
 
     - name: Install Redis backend
       run: |

dev-resources/Makefile

@@ -35,4 +35,4 @@ docs:
 docs-serve:
 	@command -v doxygen >/dev/null 2>&1 || \
 		echo "warning: 'doxygen' not found on PATH; the C++ API reference will fail to build."
-	cd $(ROOT)/.. && $(PYTHON) -m mkdocs serve -a 0.0.0.0:8001
+	cd $(ROOT)/.. && $(PYTHON) -m mkdocs serve -a 0.0.0.0:8001

dev-resources/radex-config.toml

@@ -6,7 +6,8 @@ strict_config = true
 strict_markers = true
 markers = [
     "slow: Test may be slow to run",
-    "compiled: Test has a component compiled at run time"
+    "compiled: Test has a component compiled at run time",
+    "example: Test is running one of the examples",
 ]
 
 [tool.black]
@@ -15,4 +16,4 @@ markers = [
 profile = "black"
 py_version = 312
 src_paths = ["../src/python/src", "../tests", "../example"]
-known_first_party = ["radex"]
+known_first_party = ["radex"]

example/active-learn/cpp-mpi-with-ddict/driver.py

@@ -65,7 +65,7 @@
 MAX_ITER: int = 10  # hard cap on iterations
 
 # ── Consts ─────────────────────────────────────────────────────────────────────
-HERE = Path(__file__).parent.absolute()
+HERE = Path(__file__).resolve().parent
 ROOT = HERE.parent.parent.parent
 EXAMPLES_BIN_DIR = ROOT / "install" / "bin" / "examples"
 

example/cpp-exchange/dragon/driver.py

@@ -5,7 +5,7 @@
 from dragon.data.ddict import DDict
 from dragon.native.process import Process, ProcessTemplate
 
-HERE = pathlib.Path(__file__).parent.absolute()
+HERE = pathlib.Path(__file__).resolve().parent
 ROOT = HERE.parent.parent.parent
 EXAMPLES_BIN_DIR = ROOT / "install" / "bin" / "examples"
 

example/cpp-exchange/dragon/requirements.txt

@@ -0,0 +1 @@
+dragonhpc[telemetry]>=0.14.1

example/cpp-exchange/dragon/result.txt

@@ -0,0 +1,23 @@
+==> Running Producer...
+===========================
+Hello World from Producer!!
+---------------------------
+---------------------------
+Goodbye from Producer
+===========================
+==> Producer Joined
+==> Running Consumer...
+===========================
+Hello World from Consumer!!
+---------------------------
+Some Int:          123
+Some Float:        1.23
+Some Int Tensor:   [ 1, 2, 3, 4, 5, 6, 7, 8, ]
+      \ -> Dims:   [ 2, 4, ]
+Some Float Tensor: [ 0.120000, 3.450000, 6.780000, 9.123000, ]
+        \ -> Dims: [ 4, ]
+---------------------------
+Goodbye from Consumer!!
+===========================
+==> Consumer Joined
++++ head proc exited, code 0

example/cpp-exchange/in-mem/driver.py

@@ -1,7 +1,7 @@
 import pathlib
 import subprocess
 
-HERE = pathlib.Path(__file__).parent.absolute()
+HERE = pathlib.Path(__file__).resolve().parent
 ROOT = HERE.parent.parent.parent
 EXAMPLES_BIN_DIR = ROOT / "install" / "bin" / "examples"
 

example/cpp-exchange/in-mem/result.txt

@@ -0,0 +1,11 @@
+=============
+Hello World!!
+=============
+Some Int:           123
+Some Double:        1.23
+
+Some Int Tensor:    [ 0, 1, 2, 3, ]
+        \- Dims:    [ 2, 2, ]
+Some Double Tensor: [ 0.100000, 2.300000, 4.500000, 6.700000, ]
+           \- Dims: [ 4, ]
+=============

example/py-cpp-exchange/dragon/app.cpp

@@ -46,6 +46,7 @@ void print_vector_key(radex::IClient &client, const std::string &key) {
 }
 
 int main() {
+    const auto DELAY_SET_TIME = 1'000ms;
     char *serialized_dd = getenv("SERIALIZED_DDICT");
     if (serialized_dd == nullptr) {
         throw std::runtime_error("DDict descriptor not found!");
@@ -62,22 +63,22 @@ int main() {
     print_vector_key<int>(client, "py-int-tensor");
     print_vector_key<double>(client, "py-float-tensor");
 
-    std::this_thread::sleep_for(3'000ms);
+    std::this_thread::sleep_for(DELAY_SET_TIME);
     std::cout << IDENT << "App: Setting Double" << std::endl;
     client.put_scalar<double>(radex::data::OutgoingHandle{"cpp-double"}, 1.23);
 
-    std::this_thread::sleep_for(3'000ms);
+    std::this_thread::sleep_for(DELAY_SET_TIME);
     std::cout << IDENT << "App: Setting Int" << std::endl;
     client.put_scalar<int>(radex::data::OutgoingHandle{"cpp-int"}, 987);
 
-    std::this_thread::sleep_for(3'000ms);
+    std::this_thread::sleep_for(DELAY_SET_TIME);
     std::cout << IDENT << "App: Setting Double Tensor" << std::endl;
     std::vector<double> v(12);
     std::iota(v.begin(), v.end(), 0);
     client.put_tensor(radex::data::OutgoingHandle{"cpp-double-tensor"}, {4, 3},
                       v);
 
-    std::this_thread::sleep_for(3'000ms);
+    std::this_thread::sleep_for(DELAY_SET_TIME);
     std::cout << IDENT << "App: Setting Long Tensor" << std::endl;
     client.put_tensor<int32_t>(radex::data::OutgoingHandle{"cpp-long-tensor"},
                                {2, 2, 2}, {1, 2, 3, 4, 5, 6, 7, 8});

example/py-cpp-exchange/dragon/driver.py

@@ -12,50 +12,57 @@
 from radex.clients.core import DragonClient as Client
 from radex.handles.handles import IncomingHandle, OutgoingHandle
 
-HERE = pathlib.Path(__file__).parent.absolute()
+HERE = pathlib.Path(__file__).resolve().parent
 ROOT = HERE.parent.parent.parent
 EXAMPLES_BIN_DIR = ROOT / "install" / "bin" / "examples"
+DELAY_SET_TIME = 1
 
 
 def main() -> int:
-    dd = DDict(managers_per_node=1, n_nodes=1, trace=False)
+    dd = DDict(
+        managers_per_node=1,
+        n_nodes=1,
+        trace=False,
+        wait_for_keys=True,
+        working_set_size=3,
+    )
     serial_dd = dd.serialize()
     app_tmpl = ProcessTemplate(
         target=os.fspath(EXAMPLES_BIN_DIR / "dragon-cpp-with-py"),
         env={"SERIALIZED_DDICT": serial_dd},
     )
     app = Process.from_template(app_tmpl)
 
-    print(f"Driver: Making client")
+    print(f"Driver: Making client", flush=True)
     client = Client(serial_dd, 5)
 
-    print(f"Driver: Starting app")
+    print(f"Driver: Starting app", flush=True)
     app.start()
     try:
-        time.sleep(3)
-        print("Driver: Setting Int")
+        time.sleep(DELAY_SET_TIME)
+        print("Driver: Setting Int", flush=True)
         client.put_scalar(OutgoingHandle("py-int"), 123)
 
-        time.sleep(3)
-        print("Driver: Setting Double")
+        time.sleep(DELAY_SET_TIME)
+        print("Driver: Setting Double", flush=True)
         client.put_scalar(OutgoingHandle("py-double"), 9.87)
 
-        time.sleep(3)
-        print("Driver: Setting Numpy Int")
+        time.sleep(DELAY_SET_TIME)
+        print("Driver: Setting Numpy Int", flush=True)
         client.put_scalar(OutgoingHandle("py-np-float"), np.float32(45.6))
 
-        time.sleep(3)
-        print("Driver: Setting Int Tensor")
+        time.sleep(DELAY_SET_TIME)
+        print("Driver: Setting Int Tensor", flush=True)
         client.put_tensor(OutgoingHandle("py-int-tensor"), np.arange(4, dtype=np.int32))
 
-        time.sleep(3)
-        print("Driver: Setting Float Tensor")
+        time.sleep(DELAY_SET_TIME)
+        print("Driver: Setting Float Tensor", flush=True)
         client.put_tensor(
             OutgoingHandle("py-float-tensor"),
             np.arange(12, dtype=np.float64).reshape((6, 2)),
         )
 
-        print(f"Driver: Looking for keys")
+        print(f"Driver: Looking for keys", flush=True)
         print_scalar(client, "cpp-double")
         print_scalar(client, "cpp-int")
         print_tensor(client, "cpp-double-tensor")
@@ -65,34 +72,40 @@ def main() -> int:
         py_obj_key = "my-py-obj"
         obj = C("spam-and-eggs")
         client.put_picklable(py_obj_key, obj)
-        print("Driver: Getting a py object")
+        print("Driver: Getting a py object", flush=True)
         recv = client.get_picklable(py_obj_key)
-        print(f"Driver: Got object `{recv}`")
+        print(f"Driver: Got object `{recv}`", flush=True)
     finally:
         app.join()
 
     return 0
 
 
 def print_scalar(client, key):
-    print(f"Driver: Waiting for scalar key `{key}`")
+    print(f"Driver: Waiting for scalar key `{key}`", flush=True)
     scalar = client.wait_for_scalar(IncomingHandle(key), 10)
-    print(textwrap.dedent(f"""\
+    print(
+        textwrap.dedent(f"""\
         Driver: Got scalar:
                 |- Type: {scalar.dtype}
                 \\- Value: {scalar}
-        """))
+        """),
+        flush=True,
+    )
 
 
 def print_tensor(client, key):
-    print(f"Driver: Waiting for tensor key `{key}`")
+    print(f"Driver: Waiting for tensor key `{key}`", flush=True)
     tensor = client.wait_for_tensor(IncomingHandle(key), 10)
-    print(textwrap.dedent(f"""\
+    print(
+        textwrap.dedent(f"""\
         Driver: Got tensor:
                 |- Type: {tensor.dtype}
                 |- Dims: {tensor.shape}
                 \\- Data: {tensor.ravel()}
-        """))
+        """),
+        flush=True,
+    )
 
 
 @dataclasses.dataclass(frozen=True)

example/py-cpp-exchange/dragon/requirements.txt

@@ -0,0 +1 @@
+dragonhpc[telemetry]>=0.14.1

example/py-cpp-exchange/dragon/result.txt

@@ -0,0 +1,59 @@
+Driver: Making client
+Driver: Starting app
+               App: Creating client
+               App: Client created
+               App: Waiting for scalar key `py-int`
+Driver: Setting Int
+               App: Got key `py-int` has value 123
+
+               App: Waiting for scalar key `py-double`
+Driver: Setting Double
+               App: Got key `py-double` has value 9.87
+
+               App: Waiting for scalar key `py-np-float`
+Driver: Setting Numpy Int
+               App: Got key `py-np-float` has value 45.6
+
+               App: Waiting for tensor key `py-int-tensor`
+Driver: Setting Int Tensor
+               App: Got key `py-int-tensor`
+                     |- Data: [ 0, 1, 2, 3, ]
+                     \- Dims: [ 4, ]
+
+               App: Waiting for tensor key `py-float-tensor`
+Driver: Setting Float Tensor
+Driver: Looking for keys
+Driver: Waiting for scalar key `cpp-double`
+               App: Got key `py-float-tensor`
+                     |- Data: [ 0.000000, 1.000000, 2.000000, 3.000000, 4.000000, 5.000000, 6.000000, 7.000000, 8.000000, 9.000000, 10.000000, 11.000000, ]
+                     \- Dims: [ 6, 2, ]
+
+               App: Setting Double
+Driver: Got scalar:
+        |- Type: float64
+        \- Value: 1.23
+
+Driver: Waiting for scalar key `cpp-int`
+               App: Setting Int
+Driver: Got scalar:
+        |- Type: int32
+        \- Value: 987
+
+Driver: Waiting for tensor key `cpp-double-tensor`
+               App: Setting Double Tensor
+Driver: Got tensor:
+        |- Type: float64
+        |- Dims: (4, 3)
+        \- Data: [ 0.  1.  2.  3.  4.  5.  6.  7.  8.  9. 10. 11.]
+
+Driver: Waiting for tensor key `cpp-long-tensor`
+               App: Setting Long Tensor
+Driver: Got tensor:
+        |- Type: int32
+        |- Dims: (2, 2, 2)
+        \- Data: [1 2 3 4 5 6 7 8]
+
+Driver: Setting a py object
+Driver: Getting a py object
+Driver: Got object `C(msg='spam-and-eggs')`
++++ head proc exited, code 0