Refactor thunk to native async/await and fix brace expansion

.gitignore

@@ -1,3 +1,6 @@
+# Widip
+*.jpg
+
 # VS code
 .vscode
 
@@ -64,6 +67,7 @@ cover/
 
 # Django stuff:
 *.log
+!tests/*.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal

Makefile

@@ -0,0 +1,13 @@
+JPG_FILES := $(shell git ls-files '*.jpg')
+YAML_FILES := $(JPG_FILES:.jpg=.yaml)
+
+.PHONY: all clean
+
+all: $(JPG_FILES)
+
+%.jpg: %.yaml
+	@echo "Generating $@..."
+	@echo $< | bin/yaml/shell.yaml
+
+clean:
+	rm -f $(JPG_FILES)

bin/README.md

@@ -0,0 +1,24 @@
+# Execution Model
+
+In `widip`, YAML tags such as `!eval`, `!read`, or `!print` are interpreted as **commands to execute**.
+
+## Using Executables
+
+You can use **any executable** that exists in your system's `$PATH` or by providing a relative/absolute path.
+
+**Examples:**
+
+*   **System tools:** `!python`, `!grep`, `!awk`.
+*   **Custom scripts:** `!./myscript.sh`.
+
+## Using Other YAML Files
+
+You can also use other `widip` YAML files as commands, provided they are executable (e.g., they have a valid shebang like `#!bin/widish`). This allows you to compose complex pipelines from smaller, reusable diagrams.
+
+**Example:**
+
+```yaml
+- !executable.yaml
+```
+
+This will execute the `executable.yaml` diagram as a step in your current pipeline.

bin/widish

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec python -O -m widip "$@"

bin/yaml/python.yaml

@@ -0,0 +1,2 @@
+#!bin/widish
+!python

bin/yaml/range.yaml

@@ -0,0 +1,2 @@
+#!/bin/widish
+!seq { 1, 100 }

examples/README.md

@@ -9,6 +9,8 @@ Hello world!
 
 ![](hello-world.jpg)
 
+![](hello-world.shell.jpg)
+
 ## Script
 
 ```
@@ -21,6 +23,8 @@ $ python -m widip examples/shell.yaml
 
 ![IMG](shell.jpg)
 
+![IMG](shell.shell.jpg)
+
 
 # Working with the CLI
 Open terminal and run `widip` to start an interactive session. The program `bin/yaml/shell.yaml` prompts for one command per line, so when we hit `↵ Enter` it is evaluated. When hitting `⌁ Ctrl+D` the environment exits.

pyproject.toml

@@ -10,8 +10,12 @@ name = "widip"
 version = "0.1.0"
 description = "Widip is an interactive environment for computing with wiring diagrams in modern systems"
 dependencies = [
-  "discopy>=1.2.1", "pyyaml>=6.0.1", "watchdog>=4.0.1", "nx-yaml==0.3.0",
+  "discopy>=1.2.2", "watchfiles>=1.1.1", "nx-yaml>=1.0.0",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-asyncio",
 ]
 
-[project.urls]
-"Source" = "https://github.com/colltoaction/widip"

pytest.ini

@@ -0,0 +1,3 @@
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"

tests/README.md

@@ -0,0 +1,27 @@
+# Widish Tests
+
+This directory contains test cases for the `widish` shell environment. `widish` combines the familiarity of shell commands with the structure and composability of YAML.
+
+## What is Widish?
+
+`widish` allows you to write shell scripts as YAML documents. Data flows through the structure, enabling:
+
+-   **Structured Pipelines**: Use YAML sequences (lists) to pipe data between commands.
+-   **Structured Data**: Pass structured data (like YAML mappings) between processes, not just text streams.
+-   **Composition**: Reuse YAML files as scripts/commands within other YAML scripts.
+-   **Implicit Parallelism**: Use mappings to branch data flow to multiple commands simultaneously.
+
+## Running Tests
+
+Tests are run using `pytest` and the `tests/test_harness.py` script. The harness executes each `.test.yaml` file using `bin/widish` and compares the standard output to the corresponding `.log` file.
+
+```bash
+pytest tests/test_harness.py
+```
+
+## Test Case Format
+
+Each test case consists of two files:
+
+1.  `tests/CASE.test.yaml`: The input YAML script.
+2.  `tests/CASE.log`: The expected standard output.

tests/fan-out.log

@@ -0,0 +1,2 @@
+2
+WIDIP STUFF

tests/fan-out.test.yaml

@@ -0,0 +1,4 @@
+- !echo { "widip", "project" }
+- !tr { "[:lower:]", "[:upper:]" }
+- ? !wc -w
+  ? !sed "s/PROJECT/STUFF/"

tests/git-first-commit.log

@@ -0,0 +1,2 @@
+? !id 8f20e0d66c3a6587ccd484642cea4d5db9eb9756
+? !date "Tue Feb 6 12:12:01 2024 -0300"

tests/git-first-commit.test.yaml

@@ -0,0 +1,3 @@
+!git { log, "--max-parents=0" }:
+  !grep commit: !sed "s/commit /? !id /"
+  !grep Date: !sed "s/Date:   /? !date \"/;s/$/\"/"

tests/infinite-counter.log

@@ -0,0 +1,5 @@
+1
+2
+3
+4
+5

tests/infinite-counter.test.yaml

@@ -0,0 +1,2 @@
+- !bin/yaml/range.yaml
+- !head { -n, 5 }

tests/test_harness.py

@@ -0,0 +1,37 @@
+import pytest
+import subprocess
+import glob
+import os
+
+# Find all test cases
+TEST_DIR = os.path.dirname(__file__)
+TEST_CASES = sorted(glob.glob(os.path.join(TEST_DIR, "*.test.yaml")))
+
+@pytest.mark.parametrize("test_file", TEST_CASES)
+def test_case(test_file):
+    # Determine the log file path
+    log_file = test_file.replace(".test.yaml", ".log")
+
+    # Check if log file exists
+    assert os.path.exists(log_file), f"Log file missing for {test_file}"
+
+    # Read input and expected output
+    with open(test_file, "r") as f:
+        input_content = f.read()
+
+    with open(log_file, "r") as f:
+        expected_output = f.read()
+
+    # Run the shell
+    # Assuming running from repo root
+    cmd = ["bin/widish", test_file]
+
+    result = subprocess.run(
+        cmd,
+        text=True,
+        capture_output=True,
+        check=False
+    )
+
+    # Assert output
+    assert result.stdout == expected_output

widip/__main__.py

@@ -1,13 +1,46 @@
-import sys
+if __debug__:
+    # Non-interactive backend for file output
+    import matplotlib
+    matplotlib.use('agg')
 
-# Stop starting a Matplotlib GUI
-import matplotlib
-matplotlib.use('agg')
+import argparse
+import asyncio
 
-from .watch import shell_main, widish_main
+from .interactive import async_shell_main, async_widish_main, async_command_main
+from .watch import run_with_watcher
 
 
-match sys.argv:
-    case [_]:
-        shell_main("bin/yaml/shell.yaml")
-    case [_, file_name, *args]: widish_main(file_name, *args)
+def main():
+    parser = argparse.ArgumentParser(
+        description="Widip: an interactive environment for computing with wiring diagrams"
+    )
+    parser.add_argument(
+        "-c",
+        dest="command_string",
+        help="read commands from the first non-option argument"
+    )
+    parser.add_argument(
+        "operands",
+        nargs=argparse.REMAINDER,
+        help="[command_string | file] [arguments...]"
+    )
+
+    args = parser.parse_args()
+
+    try:
+        if args.command_string is not None:
+            asyncio.run(async_command_main(args.command_string, *args.operands))
+        elif args.operands:
+            file_name = args.operands[0]
+            file_args = args.operands[1:]
+            asyncio.run(async_widish_main(file_name, *file_args))
+        else:
+            async_shell_runner = async_shell_main("bin/yaml/shell.yaml")
+            interactive_shell = run_with_watcher(async_shell_runner)
+            asyncio.run(interactive_shell)
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()

widip/compiler.py

@@ -0,0 +1,56 @@
+from discopy import closed
+
+from .computer import *
+from .yaml import *
+
+
+def compile_ar(ar):
+    if isinstance(ar, Scalar):
+        if ar.tag == "exec":
+            return Exec(ar.dom, ar.cod)
+        if ar.tag:
+            return Program(ar.tag, dom=ar.dom, cod=ar.cod).uncurry()
+        return Data(ar.dom, ar.cod)
+    if isinstance(ar, Sequence):
+        inner_diagram = ar.args[0]
+        inside_compiled = SHELL_COMPILER(inner_diagram)
+
+        if ar.dom[:1] == Language:
+            return inside_compiled
+
+        if ar.n == 2:
+            return inside_compiled >> Pair(inside_compiled.cod, ar.cod)
+
+        return inside_compiled >> Sequential(inside_compiled.cod, ar.cod)
+
+    if isinstance(ar, Mapping):
+        inner_diagram = ar.args[0]
+        inside_compiled = SHELL_COMPILER(inner_diagram)
+        return inside_compiled >> Concurrent(inside_compiled.cod, ar.cod)
+
+    if isinstance(ar, Alias):
+        return Data(ar.dom, ar.cod) >> Copy(ar.cod, 2) >> closed.Id(ar.cod) @ Discard(ar.cod)
+    if isinstance(ar, Anchor):
+        return Copy(ar.dom, 2) >> closed.Id(ar.dom) @ Discard(ar.dom)
+    return ar
+
+
+class ShellFunctor(closed.Functor):
+    def __init__(self):
+        super().__init__(
+            lambda ob: ob,
+            compile_ar,
+            dom=Yaml,
+            cod=Computation
+        )
+
+SHELL_COMPILER = ShellFunctor()
+
+
+def compile_shell_program(diagram):
+    """
+    close input parameters (constants)
+    drop outputs matching input parameters
+    all boxes are io->[io]"""
+    diagram = SHELL_COMPILER(diagram)
+    return diagram