Combine workspace and in-memory stroke transport fixes

docs/in-memory-stroke-transport.md

@@ -0,0 +1,34 @@
+# In-memory stroke transport
+
+This change removes the per-stroke image files from the Java/Python effect path.
+Instead of saving a stroke input PNG and later reading a stroke output PNG, Java
+sends the canvas image to Python through stdin and receives the processed result
+through stdout.
+
+The transport is a small length-prefixed binary protocol:
+
+```text
+Java -> Python stdin:
+[4-byte JSON length][JSON instructions][4-byte PNG length][PNG bytes]
+
+Python -> Java stdout:
+[4-byte JSON length][JSON response][4-byte PNG length][PNG bytes]
+```
+
+`stdout` is reserved for the binary response. Logs and debug output are routed to
+`stderr`, so debug prints cannot corrupt the image response.
+
+```mermaid
+flowchart LR
+    A["Java canvas PImage"] --> B["Encode PNG bytes in memory"]
+    B --> C["stdin: JSON instructions + PNG bytes"]
+    C --> D["Python apply_effect.py --stdio"]
+    D --> E["Run brush/effect in memory"]
+    E --> F["stdout: response JSON + result PNG bytes"]
+    F --> G["Java decodes result into PImage"]
+    G --> H["Apply result to canvas"]
+```
+
+The persistent stroke JSON stays lightweight and still stores the brush, user
+parameters, and path data. Legacy file-based stroke outputs remain supported as a
+fallback for older projects.

effect/GoL/GoL.py

@@ -140,7 +140,7 @@ def game_of_life(nhood):
     qc = QuantumCircuit(qr,name='conway')
     v = np.array(nhood).reshape(9,2)
     for qubit,state in enumerate(v):
-        qc.initialize(state,qubit)
+        qc.initialize(state, qubit, normalize=True)
     for q in [0,1,2,3,4,5,6,7,8]:
         pass
         if q!=4:
@@ -227,4 +227,4 @@ def extract_neighbourhoods(x, y):
     for key in after_iteration:
         image[key[0], key[1],:3] = np.round(np.array(colorsys.hsv_to_rgb(*after_iteration[key]))*255).astype(np.uint8)
     return image
-#%%
+#%%

effect/apply_effect.py

@@ -11,8 +11,9 @@
 import argparse
 import os
 import time
-from utils import *
 import contextlib
+import struct
+from io import BytesIO
 
 app_path = Path(sys.path[0] + "/..") # Path to the app folder
 
@@ -51,7 +52,48 @@ def process_variable(var_type: str, variable: any):
             raise ValueError(f"Unsupported type: {var_type}")
 
 
-def process_effect(instr: dict):
+def image_from_png_bytes(image_bytes: bytes) -> np.ndarray:
+    with Image.open(BytesIO(image_bytes)) as img:
+        return np.array(img.convert("RGBA"))
+
+
+def image_to_png_bytes(image: np.ndarray) -> bytes:
+    buffer = BytesIO()
+    Image.fromarray(image.astype(np.uint8)).save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def read_exact(stream, size: int) -> bytes:
+    chunks = []
+    remaining = size
+    while remaining > 0:
+        chunk = stream.read(remaining)
+        if not chunk:
+            raise EOFError("Unexpected end of stream while reading binary request.")
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+
+def read_binary_request(stream) -> tuple[dict, bytes]:
+    json_size = struct.unpack(">I", read_exact(stream, 4))[0]
+    instructions = json.loads(read_exact(stream, json_size).decode("utf-8"))
+
+    image_size = struct.unpack(">I", read_exact(stream, 4))[0]
+    image_bytes = read_exact(stream, image_size)
+    return instructions, image_bytes
+
+
+def write_binary_response(stream, response: dict, image_bytes: bytes = b""):
+    response_bytes = json.dumps(response).encode("utf-8")
+    stream.write(struct.pack(">I", len(response_bytes)))
+    stream.write(response_bytes)
+    stream.write(struct.pack(">I", len(image_bytes)))
+    stream.write(image_bytes)
+    stream.flush()
+
+
+def process_effect(instr: dict, image_rgba: np.ndarray | None = None):
     # Extract stroke_id and project_id from instructions
     stroke_id = instr.get("stroke_id", False)
     project_id = instr.get("project_id", False)
@@ -67,7 +109,7 @@ def process_effect(instr: dict):
     effect_path = app_path / f"effect/{effect_id}"
     req_path = effect_path / f"{effect_id}_requirements.json"
 
-    with open(req_path, 'r') as req_file:
+    with open(req_path, 'r', encoding='utf-8') as req_file:
         req = json.load(req_file)
 
     # Check dependencies with version control
@@ -79,14 +121,18 @@ def process_effect(instr: dict):
         except ImportError as e:
             raise ImportError(f"Failed to load dependency {dependency}: {e}")
 
-    # Process image
-    image_path = project_path / f"stroke/{stroke_id}_input.png"
+    # Process image. In the stdio path Java sends the canvas image in memory.
+    # The legacy file path remains as a fallback for older callers.
+    if image_rgba is not None:
+        req["stroke_input"]["image_rgba"] = image_rgba
+    else:
+        image_path = project_path / f"stroke/{stroke_id}_input.png"
 
-    if not image_path.is_file():
-        raise FileNotFoundError(f"Image file not found at {image_path}")
-    
-    with Image.open(image_path) as img:
-        req["stroke_input"]["image_rgba"] = np.array(img.convert("RGBA")) # Ensure the image is in RGBA format
+        if not image_path.is_file():
+            raise FileNotFoundError(f"Image file not found at {image_path}")
+
+        with Image.open(image_path) as img:
+            req["stroke_input"]["image_rgba"] = np.array(img.convert("RGBA")) # Ensure the image is in RGBA format
 
 
     # Process user_input and stroke_input
@@ -125,7 +171,7 @@ def process_effect(instr: dict):
 
     return req
 
-def apply_effect(req: dict):
+def apply_effect(req: dict, write_output: bool = True):
     # Save the input image to apply mask
     input_image = copy(req["stroke_input"]["image_rgba"])
 
@@ -141,11 +187,12 @@ def apply_effect(req: dict):
     mask = np.all(new_image == input_image, axis=-1)  
     new_image[mask] = [0, 0, 0, 0]  # Set differing pixels to [0, 0, 0, 0]
 
-    output_path = req["stroke_output_path"]
-    output_path.parent.mkdir(parents=True, exist_ok=True)  # Ensure the directory exists
-    Image.fromarray(new_image.astype(np.uint8)).save(output_path, format="PNG")
+    if write_output:
+        output_path = req["stroke_output_path"]
+        output_path.parent.mkdir(parents=True, exist_ok=True)  # Ensure the directory exists
+        Image.fromarray(new_image.astype(np.uint8)).save(output_path, format="PNG")
 
-    return True
+    return new_image.astype(np.uint8)
 
 def record_error(error):
     log_file = app_path / "log/error.log"
@@ -156,7 +203,7 @@ def record_error(error):
     with open(log_file, "a") as log:
         log.write(error_message)
 
-    print(error_message)    
+    print(error_message, file=sys.stderr)
 
 def dump_json(data, file_path):
     # Make sure the directory exists
@@ -191,23 +238,55 @@ def flush(self):
 
 
 if __name__ == "__main__":
-
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description="Apply an effect to a stroke in a project.")
+    parser.add_argument("stroke_path", nargs="?", type=str, help="The ID of the stroke.")
+    parser.add_argument(
+        "--stdio",
+        action="store_true",
+        help="Read an in-memory request from stdin and write JSON response to stdout.",
+    )
+    args = parser.parse_args()
+
     log_output_file = app_path / "log/console_output.log"
     log_output_file.parent.mkdir(parents=True, exist_ok=True)
     log_fh = open(log_output_file, "a")
+    binary_stdout = sys.__stdout__.buffer
 
-    sys.stdout = Tee(sys.stdout, log_fh)
-    sys.stderr = Tee(sys.stderr, log_fh)
-
-    # Parse command line arguments
-    parser = argparse.ArgumentParser(description="Apply an effect to a stroke in a project.")
-    parser.add_argument("stroke_path", type=str, help="The ID of the stroke.")
-    args = parser.parse_args()
+    if args.stdio:
+        # stdout is reserved for the binary response protocol in stdio mode.
+        # Send any debug prints from effects/dependencies to stderr instead.
+        sys.stdout = Tee(sys.stderr, log_fh)
+        sys.stderr = Tee(sys.stderr, log_fh)
+    else:
+        sys.stdout = Tee(sys.stdout, log_fh)
+        sys.stderr = Tee(sys.stderr, log_fh)
+
+    from utils import *
 
     success = False
     instructions = {}
 
     try:
+        if args.stdio:
+            instructions, image_bytes = read_binary_request(sys.stdin.buffer)
+            image_rgba = image_from_png_bytes(image_bytes)
+
+            data = process_effect(instructions, image_rgba=image_rgba)
+            result_image = apply_effect(data, write_output=False)
+            result_bytes = image_to_png_bytes(result_image)
+
+            response = {
+                "effect_success": "true",
+                "width": int(result_image.shape[1]),
+                "height": int(result_image.shape[0]),
+            }
+            write_binary_response(binary_stdout, response, result_bytes)
+            sys.exit(0)
+
+        if args.stroke_path is None:
+            raise ValueError("stroke_path is required unless --stdio is used.")
+
         if not Path(args.stroke_path).is_file():
             raise FileNotFoundError(f"Stroke file not found at {args.stroke_path}")
 
@@ -239,20 +318,23 @@ def flush(self):
 
     except FileNotFoundError as e:
         record_error(e)
-        if instructions:
+        if instructions and not args.stdio:
             instructions["effect_received"] = False
             dump_json(instructions, args.stroke_path)
         success = False
     except Exception as e:
         record_error(e)
-        if instructions:
+        if instructions and not args.stdio:
             instructions["effect_success"] = False
             dump_json(instructions, args.stroke_path)
         success = False
         # Redirect stdout and stderr to a log file
 
 
-    if success:
+    if args.stdio:
+        write_binary_response(binary_stdout, {"effect_success": "false"})
+        sys.exit(1)
+    elif success:
         print("Effect applied successfully.")
         sys.exit(0)
     else:

effect/clone/clone.py

@@ -71,19 +71,27 @@ def run(params):
     height = image.shape[0]
     width = image.shape[1]
 
-    # Extract the copy and past points
+    # Extract the copy and paste points
     clicks = params["stroke_input"]["clicks"]
-    assert len(clicks) == 2, "The number of clicks must 2, i.e. copy and paste"
+    path = params["stroke_input"]["path"]
+    if len(clicks) == 1 and len(path) > 1:
+        clicks = np.array([clicks[0], path[-1]])
+    elif len(clicks) > 2:
+        clicks = np.array([clicks[0], clicks[-1]])
+    assert len(clicks) >= 2, "The number of clicks must be at least 2, i.e. copy and paste"
 
     offset = clicks[1]-clicks[0]
 
     # Extract the lasso path
-    path = params["stroke_input"]["path"]
 
     # Remove any leftover points
-    while np.all(path[-1] != clicks[-1]):
+    while len(path) > 0 and np.all(path[-1] != clicks[-1]):
         path = path[:-1]
-    path = path[:-1] #Remove the last click
+    if len(path) > 0:
+        path = path[:-1] #Remove the last click
+
+    if len(path) == 0:
+        path = params["stroke_input"]["path"]
 
     # Create the region around those points
     copy_region = utils.points_within_lasso(path, border = (height, width))
@@ -127,6 +135,10 @@ def run(params):
 
 
     paste_region = utils.points_within_lasso(path + offset, border = (height, width))
-    image[paste_region[:, 0], paste_region[:, 1],:3] = (paste_selection * 255).astype(np.uint8)
+    paste_count = min(len(paste_region), len(paste_selection))
+    if paste_count > 0:
+        paste_region = paste_region[:paste_count]
+        paste_selection = paste_selection[:paste_count]
+        image[paste_region[:, 0], paste_region[:, 1],:3] = (paste_selection * 255).astype(np.uint8)
 
     return image

effect/heisenbrush2/heisenbrush2.py

@@ -99,9 +99,9 @@ def run_heisenberg_hardware(dt_list, radius, phi, theta):
         observables = get_mean_magnetization(n_qubits)
 
         # Run the estimator
-        values=utils.run_estimator(circuits, observables, backend=None)
-
-        values=np.array([val[0] for val in values])
+        values = np.asarray(utils.run_estimator(circuits, observables, backend=None))
+        if values.ndim > 1:
+            values = values[:, 0]
 
         print(f"Values: {values}")
         return values
@@ -173,4 +173,4 @@ def run(params):
         image[region[:, 0], region[:, 1]] = utils.apply_patch_to_image(image[region[:, 0], region[:, 1]], new_patch)
 
     print("Heisenberg effect applied successfully")
-    return image
+    return image