diff --git a/docs/in-memory-stroke-transport.md b/docs/in-memory-stroke-transport.md new file mode 100644 index 0000000..3c474b1 --- /dev/null +++ b/docs/in-memory-stroke-transport.md @@ -0,0 +1,34 @@ +# In-memory stroke transport + +This change removes the per-stroke image files from the Java/Python effect path. +Instead of saving a stroke input PNG and later reading a stroke output PNG, Java +sends the canvas image to Python through stdin and receives the processed result +through stdout. + +The transport is a small length-prefixed binary protocol: + +```text +Java -> Python stdin: +[4-byte JSON length][JSON instructions][4-byte PNG length][PNG bytes] + +Python -> Java stdout: +[4-byte JSON length][JSON response][4-byte PNG length][PNG bytes] +``` + +`stdout` is reserved for the binary response. Logs and debug output are routed to +`stderr`, so debug prints cannot corrupt the image response. + +```mermaid +flowchart LR + A["Java canvas PImage"] --> B["Encode PNG bytes in memory"] + B --> C["stdin: JSON instructions + PNG bytes"] + C --> D["Python apply_effect.py --stdio"] + D --> E["Run brush/effect in memory"] + E --> F["stdout: response JSON + result PNG bytes"] + F --> G["Java decodes result into PImage"] + G --> H["Apply result to canvas"] +``` + +The persistent stroke JSON stays lightweight and still stores the brush, user +parameters, and path data. Legacy file-based stroke outputs remain supported as a +fallback for older projects. diff --git a/effect/GoL/GoL.py b/effect/GoL/GoL.py index e61a920..e6e210c 100644 --- a/effect/GoL/GoL.py +++ b/effect/GoL/GoL.py @@ -140,7 +140,7 @@ def game_of_life(nhood): qc = QuantumCircuit(qr,name='conway') v = np.array(nhood).reshape(9,2) for qubit,state in enumerate(v): - qc.initialize(state,qubit) + qc.initialize(state, qubit, normalize=True) for q in [0,1,2,3,4,5,6,7,8]: pass if q!=4: @@ -227,4 +227,4 @@ def extract_neighbourhoods(x, y): for key in after_iteration: image[key[0], key[1],:3] = np.round(np.array(colorsys.hsv_to_rgb(*after_iteration[key]))*255).astype(np.uint8) return image -#%% \ No newline at end of file +#%% diff --git a/effect/apply_effect.py b/effect/apply_effect.py index 60d46c4..c56fadb 100644 --- a/effect/apply_effect.py +++ b/effect/apply_effect.py @@ -11,8 +11,9 @@ import argparse import os import time -from utils import * import contextlib +import struct +from io import BytesIO app_path = Path(sys.path[0] + "/..") # Path to the app folder @@ -51,7 +52,48 @@ def process_variable(var_type: str, variable: any): raise ValueError(f"Unsupported type: {var_type}") -def process_effect(instr: dict): +def image_from_png_bytes(image_bytes: bytes) -> np.ndarray: + with Image.open(BytesIO(image_bytes)) as img: + return np.array(img.convert("RGBA")) + + +def image_to_png_bytes(image: np.ndarray) -> bytes: + buffer = BytesIO() + Image.fromarray(image.astype(np.uint8)).save(buffer, format="PNG") + return buffer.getvalue() + + +def read_exact(stream, size: int) -> bytes: + chunks = [] + remaining = size + while remaining > 0: + chunk = stream.read(remaining) + if not chunk: + raise EOFError("Unexpected end of stream while reading binary request.") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def read_binary_request(stream) -> tuple[dict, bytes]: + json_size = struct.unpack(">I", read_exact(stream, 4))[0] + instructions = json.loads(read_exact(stream, json_size).decode("utf-8")) + + image_size = struct.unpack(">I", read_exact(stream, 4))[0] + image_bytes = read_exact(stream, image_size) + return instructions, image_bytes + + +def write_binary_response(stream, response: dict, image_bytes: bytes = b""): + response_bytes = json.dumps(response).encode("utf-8") + stream.write(struct.pack(">I", len(response_bytes))) + stream.write(response_bytes) + stream.write(struct.pack(">I", len(image_bytes))) + stream.write(image_bytes) + stream.flush() + + +def process_effect(instr: dict, image_rgba: np.ndarray | None = None): # Extract stroke_id and project_id from instructions stroke_id = instr.get("stroke_id", False) project_id = instr.get("project_id", False) @@ -67,7 +109,7 @@ def process_effect(instr: dict): effect_path = app_path / f"effect/{effect_id}" req_path = effect_path / f"{effect_id}_requirements.json" - with open(req_path, 'r') as req_file: + with open(req_path, 'r', encoding='utf-8') as req_file: req = json.load(req_file) # Check dependencies with version control @@ -79,14 +121,18 @@ def process_effect(instr: dict): except ImportError as e: raise ImportError(f"Failed to load dependency {dependency}: {e}") - # Process image - image_path = project_path / f"stroke/{stroke_id}_input.png" + # Process image. In the stdio path Java sends the canvas image in memory. + # The legacy file path remains as a fallback for older callers. + if image_rgba is not None: + req["stroke_input"]["image_rgba"] = image_rgba + else: + image_path = project_path / f"stroke/{stroke_id}_input.png" - if not image_path.is_file(): - raise FileNotFoundError(f"Image file not found at {image_path}") - - with Image.open(image_path) as img: - req["stroke_input"]["image_rgba"] = np.array(img.convert("RGBA")) # Ensure the image is in RGBA format + if not image_path.is_file(): + raise FileNotFoundError(f"Image file not found at {image_path}") + + with Image.open(image_path) as img: + req["stroke_input"]["image_rgba"] = np.array(img.convert("RGBA")) # Ensure the image is in RGBA format # Process user_input and stroke_input @@ -125,7 +171,7 @@ def process_effect(instr: dict): return req -def apply_effect(req: dict): +def apply_effect(req: dict, write_output: bool = True): # Save the input image to apply mask input_image = copy(req["stroke_input"]["image_rgba"]) @@ -141,11 +187,12 @@ def apply_effect(req: dict): mask = np.all(new_image == input_image, axis=-1) new_image[mask] = [0, 0, 0, 0] # Set differing pixels to [0, 0, 0, 0] - output_path = req["stroke_output_path"] - output_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the directory exists - Image.fromarray(new_image.astype(np.uint8)).save(output_path, format="PNG") + if write_output: + output_path = req["stroke_output_path"] + output_path.parent.mkdir(parents=True, exist_ok=True) # Ensure the directory exists + Image.fromarray(new_image.astype(np.uint8)).save(output_path, format="PNG") - return True + return new_image.astype(np.uint8) def record_error(error): log_file = app_path / "log/error.log" @@ -156,7 +203,7 @@ def record_error(error): with open(log_file, "a") as log: log.write(error_message) - print(error_message) + print(error_message, file=sys.stderr) def dump_json(data, file_path): # Make sure the directory exists @@ -191,23 +238,55 @@ def flush(self): if __name__ == "__main__": - + # Parse command line arguments + parser = argparse.ArgumentParser(description="Apply an effect to a stroke in a project.") + parser.add_argument("stroke_path", nargs="?", type=str, help="The ID of the stroke.") + parser.add_argument( + "--stdio", + action="store_true", + help="Read an in-memory request from stdin and write JSON response to stdout.", + ) + args = parser.parse_args() + log_output_file = app_path / "log/console_output.log" log_output_file.parent.mkdir(parents=True, exist_ok=True) log_fh = open(log_output_file, "a") + binary_stdout = sys.__stdout__.buffer - sys.stdout = Tee(sys.stdout, log_fh) - sys.stderr = Tee(sys.stderr, log_fh) - - # Parse command line arguments - parser = argparse.ArgumentParser(description="Apply an effect to a stroke in a project.") - parser.add_argument("stroke_path", type=str, help="The ID of the stroke.") - args = parser.parse_args() + if args.stdio: + # stdout is reserved for the binary response protocol in stdio mode. + # Send any debug prints from effects/dependencies to stderr instead. + sys.stdout = Tee(sys.stderr, log_fh) + sys.stderr = Tee(sys.stderr, log_fh) + else: + sys.stdout = Tee(sys.stdout, log_fh) + sys.stderr = Tee(sys.stderr, log_fh) + + from utils import * success = False instructions = {} try: + if args.stdio: + instructions, image_bytes = read_binary_request(sys.stdin.buffer) + image_rgba = image_from_png_bytes(image_bytes) + + data = process_effect(instructions, image_rgba=image_rgba) + result_image = apply_effect(data, write_output=False) + result_bytes = image_to_png_bytes(result_image) + + response = { + "effect_success": "true", + "width": int(result_image.shape[1]), + "height": int(result_image.shape[0]), + } + write_binary_response(binary_stdout, response, result_bytes) + sys.exit(0) + + if args.stroke_path is None: + raise ValueError("stroke_path is required unless --stdio is used.") + if not Path(args.stroke_path).is_file(): raise FileNotFoundError(f"Stroke file not found at {args.stroke_path}") @@ -239,20 +318,23 @@ def flush(self): except FileNotFoundError as e: record_error(e) - if instructions: + if instructions and not args.stdio: instructions["effect_received"] = False dump_json(instructions, args.stroke_path) success = False except Exception as e: record_error(e) - if instructions: + if instructions and not args.stdio: instructions["effect_success"] = False dump_json(instructions, args.stroke_path) success = False # Redirect stdout and stderr to a log file - if success: + if args.stdio: + write_binary_response(binary_stdout, {"effect_success": "false"}) + sys.exit(1) + elif success: print("Effect applied successfully.") sys.exit(0) else: diff --git a/effect/clone/clone.py b/effect/clone/clone.py index ee7a267..6c27bfb 100644 --- a/effect/clone/clone.py +++ b/effect/clone/clone.py @@ -71,19 +71,27 @@ def run(params): height = image.shape[0] width = image.shape[1] - # Extract the copy and past points + # Extract the copy and paste points clicks = params["stroke_input"]["clicks"] - assert len(clicks) == 2, "The number of clicks must 2, i.e. copy and paste" + path = params["stroke_input"]["path"] + if len(clicks) == 1 and len(path) > 1: + clicks = np.array([clicks[0], path[-1]]) + elif len(clicks) > 2: + clicks = np.array([clicks[0], clicks[-1]]) + assert len(clicks) >= 2, "The number of clicks must be at least 2, i.e. copy and paste" offset = clicks[1]-clicks[0] # Extract the lasso path - path = params["stroke_input"]["path"] # Remove any leftover points - while np.all(path[-1] != clicks[-1]): + while len(path) > 0 and np.all(path[-1] != clicks[-1]): path = path[:-1] - path = path[:-1] #Remove the last click + if len(path) > 0: + path = path[:-1] #Remove the last click + + if len(path) == 0: + path = params["stroke_input"]["path"] # Create the region around those points copy_region = utils.points_within_lasso(path, border = (height, width)) @@ -127,6 +135,10 @@ def run(params): paste_region = utils.points_within_lasso(path + offset, border = (height, width)) - image[paste_region[:, 0], paste_region[:, 1],:3] = (paste_selection * 255).astype(np.uint8) + paste_count = min(len(paste_region), len(paste_selection)) + if paste_count > 0: + paste_region = paste_region[:paste_count] + paste_selection = paste_selection[:paste_count] + image[paste_region[:, 0], paste_region[:, 1],:3] = (paste_selection * 255).astype(np.uint8) return image diff --git a/effect/heisenbrush2/heisenbrush2.py b/effect/heisenbrush2/heisenbrush2.py index 226d8e3..d61eb3b 100644 --- a/effect/heisenbrush2/heisenbrush2.py +++ b/effect/heisenbrush2/heisenbrush2.py @@ -99,9 +99,9 @@ def run_heisenberg_hardware(dt_list, radius, phi, theta): observables = get_mean_magnetization(n_qubits) # Run the estimator - values=utils.run_estimator(circuits, observables, backend=None) - - values=np.array([val[0] for val in values]) + values = np.asarray(utils.run_estimator(circuits, observables, backend=None)) + if values.ndim > 1: + values = values[:, 0] print(f"Values: {values}") return values @@ -173,4 +173,4 @@ def run(params): image[region[:, 0], region[:, 1]] = utils.apply_patch_to_image(image[region[:, 0], region[:, 1]], new_patch) print("Heisenberg effect applied successfully") - return image \ No newline at end of file + return image diff --git a/effect/utils.py b/effect/utils.py index 624c17c..e44f8b5 100644 --- a/effect/utils.py +++ b/effect/utils.py @@ -1,13 +1,28 @@ import numpy as np -from qiskit import generate_preset_pass_manager -from qiskit.primitives import BackendEstimatorV2 as Estimator import os -from qiskit_aer import AerSimulator import colorsys from itertools import product -import matplotlib.pyplot as plt -from matplotlib.path import Path as plt_path -from scipy.ndimage import distance_transform_edt + +try: + from matplotlib.path import Path as plt_path +except ImportError: + plt_path = None + +try: + from scipy.ndimage import distance_transform_edt +except ImportError: + distance_transform_edt = None + +try: + from qiskit import generate_preset_pass_manager + from qiskit.primitives import BackendEstimatorV2 as Estimator + from qiskit.quantum_info import Statevector + from qiskit_aer import AerSimulator +except ImportError: + generate_preset_pass_manager = None + Estimator = None + Statevector = None + AerSimulator = None def svd(matrix=None,U=None,S=None,Vt=None): if U is not None: @@ -89,21 +104,39 @@ def points_within_radius(points, radius=10, border = None, return_distance = Fal # Create binary mask mask = np.zeros((height, width), dtype=bool) mask[shifted[:, 0], shifted[:, 1]] = True # y, x - - # Distance transform - dist = distance_transform_edt(~mask) - # Find pixels within offset - region_mask = dist <= radius - ys, xs = np.nonzero(region_mask) - - # Shift back to original coordinates - coords = np.stack([ys, xs], axis=1) + min_yx - #print(coords) + + if distance_transform_edt is not None: + # Distance transform + dist = distance_transform_edt(~mask) + # Find pixels within offset + region_mask = dist <= radius + ys, xs = np.nonzero(region_mask) + + # Shift back to original coordinates + coords = np.stack([ys, xs], axis=1) + min_yx + distances = dist[ys, xs] / radius + else: + line_points = np.argwhere(mask) + grid_y, grid_x = np.indices((height, width)) + grid = np.stack([grid_y.ravel(), grid_x.ravel()], axis=1) + + min_dist_sq = np.empty(grid.shape[0], dtype=np.float64) + chunk_size = 2048 + for start in range(0, grid.shape[0], chunk_size): + end = start + chunk_size + chunk = grid[start:end] + diff = chunk[:, None, :] - line_points[None, :, :] + dist_sq = np.sum(diff * diff, axis=2) + min_dist_sq[start:end] = dist_sq.min(axis=1) + + region_mask = min_dist_sq <= radius ** 2 + coords = grid[region_mask] + min_yx + distances = np.sqrt(min_dist_sq[region_mask]) / radius + if border is not None: coords = np.clip(coords, [0, 0], [border[0] - 1, border[1] - 1]) if return_distance: - distances = dist[ys, xs] / radius return coords, distances return coords @@ -115,10 +148,24 @@ def points_within_lasso(points,border = None): max_y = np.max(points[:,0])+1 grid = list(product(np.arange(min_y,max_y), np.arange(min_x,max_x))) - # Create path from polygon - path = plt_path(points) - # Test which points are inside the path - mask = path.contains_points(grid) + if plt_path is not None: + # Create path from polygon + path = plt_path(points) + # Test which points are inside the path + mask = path.contains_points(grid) + else: + x = np.array([point[1] for point in grid], dtype=float) + y = np.array([point[0] for point in grid], dtype=float) + poly_x = points[:, 1].astype(float) + poly_y = points[:, 0].astype(float) + mask = np.zeros(len(grid), dtype=bool) + j = len(points) - 1 + for i in range(len(points)): + intersects = ((poly_y[i] > y) != (poly_y[j] > y)) & ( + x < (poly_x[j] - poly_x[i]) * (y - poly_y[i]) / (poly_y[j] - poly_y[i] + 1e-12) + poly_x[i] + ) + mask ^= intersects + j = i # Get the pixel coordinates that are inside result = np.array(grid)[mask] @@ -152,7 +199,31 @@ def run_estimator(circuits, operators, backend=None, options = None): It can receive a single circuit or a list of circuits. It can receive a single operator or a list of operators or a list of list of operators (one for each circuit). ''' - + if Statevector is None: + raise ImportError("Qiskit and qiskit-aer are required to run estimator-based effects.") + + circuit_list = circuits if isinstance(circuits, list) else [circuits] + n_circuits = len(circuit_list) + + if isinstance(operators, list): + if operators and isinstance(operators[0], list): + assert len(operators) == n_circuits, "Number of circuits and operators must match" + operator_lists = operators + else: + operator_lists = [operators for _ in range(n_circuits)] + else: + operator_lists = [[operators] for _ in range(n_circuits)] + + obs = [] + for circuit, ops in zip(circuit_list, operator_lists): + state = Statevector.from_instruction(circuit) + obs.append(np.array([np.real(state.expectation_value(op)) for op in ops])) + + if n_circuits == 1: + return obs[0] + + return obs + #iqm_server_url = "https://cocos.resonance.meetiqm.com/garnet:mock" # Replace this with the correct URL #provider = IQMProvider(iqm_server_url) #backend = provider.get_backend('garnet') @@ -352,4 +423,3 @@ def apply_patch_to_image(original_image: np.ndarray, new_patch: np.ndarray, blur new_patch[..., :3] = (1 - alpha) * original_float[..., :3] + alpha * new_patch[..., :3] return (new_patch * 255).astype(np.uint8) - \ No newline at end of file diff --git a/src/EffectManager.java b/src/EffectManager.java index 886286c..11ff09c 100644 --- a/src/EffectManager.java +++ b/src/EffectManager.java @@ -17,7 +17,7 @@ public void loadEffects() { // Clear existing effects to prevent contamination effects.clear(); - File effectsDir = new File("effect"); + File effectsDir = QuantumBrush.appFile("effect"); if (!effectsDir.exists() || !effectsDir.isDirectory()) { System.err.println("Error: effect directory not found or not a directory"); return; diff --git a/src/FileManager.java b/src/FileManager.java index b48be3f..58480b3 100644 --- a/src/FileManager.java +++ b/src/FileManager.java @@ -18,22 +18,23 @@ public FileManager(QuantumBrush app) { public void saveProject(String projectId, PImage originalImage) { String projectPath = "project/" + projectId; ensureDirectoryExists(projectPath); + File projectDir = QuantumBrush.appFile(projectPath); // Save original image if (originalImage != null) { - originalImage.save(projectPath + "/original.png"); + originalImage.save(new File(projectDir, "original.png").getAbsolutePath()); } // IMPORTANT: Also save the current state (with all effects applied) PImage currentImage = app.getCurrentImage(); if (currentImage != null) { - currentImage.save(projectPath + "/current.png"); + currentImage.save(new File(projectDir, "current.png").getAbsolutePath()); System.out.println("Saved current state to: " + projectPath + "/current.png"); } } public boolean ensureDirectoryExists(String path) { - File directory = new File(path); + File directory = QuantumBrush.appFile(path); if (!directory.exists()) { return directory.mkdirs(); } @@ -51,17 +52,17 @@ public void saveStroke( // Save instructions String instructionsPath = strokeDirPath + "/" + strokeId + "_instructions.json"; - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, QuantumBrush.appFile(instructionsPath).getAbsolutePath()); // Save input image if (inputImage != null) { String imagePath = strokeDirPath + "/" + strokeId + "_input.png"; - inputImage.save(imagePath); + inputImage.save(QuantumBrush.appFile(imagePath).getAbsolutePath()); } } public JSONObject loadEffectRequirements(String effectName) { - File requirementsFile = new File( + File requirementsFile = QuantumBrush.appFile( "effect/" + effectName + "/" + effectName + "_requirements.json" ); if (requirementsFile.exists()) { @@ -89,18 +90,18 @@ public void createProjectMetadata(String projectId, String projectName) { metadata.setLong("modified_time", currentTime); // Save metadata - app.saveJSONObject(metadata, "metadata/" + projectId + ".json"); + app.saveJSONObject(metadata, QuantumBrush.appFile("metadata/" + projectId + ".json").getAbsolutePath()); } public void updateProjectMetadata(String projectId) { String metadataPath = "metadata/" + projectId + ".json"; - File metadataFile = new File(metadataPath); + File metadataFile = QuantumBrush.appFile(metadataPath); if (metadataFile.exists()) { try { - JSONObject metadata = app.loadJSONObject(metadataPath); + JSONObject metadata = app.loadJSONObject(metadataFile.getAbsolutePath()); metadata.setLong("modified_time", System.currentTimeMillis()); - app.saveJSONObject(metadata, metadataPath); + app.saveJSONObject(metadata, metadataFile.getAbsolutePath()); } catch (Exception e) { System.err.println("Error updating metadata: " + e.getMessage()); } @@ -110,7 +111,7 @@ public void updateProjectMetadata(String projectId) { public ArrayList getProjectsMetadata() { ArrayList projects = new ArrayList<>(); - File metadataDir = new File("metadata"); + File metadataDir = QuantumBrush.appFile("metadata"); if (!metadataDir.exists()) { ensureDirectoryExists("metadata"); return projects; @@ -126,7 +127,7 @@ public ArrayList getProjectsMetadata() { // Check if the corresponding project directory exists JSONObject metadata = app.loadJSONObject(file.getAbsolutePath()); String projectId = metadata.getString("project_id", ""); - File projectDir = new File("project/" + projectId); + File projectDir = QuantumBrush.appFile("project/" + projectId); // Only add to the list if the project directory exists if (projectDir.exists() && projectDir.isDirectory()) { @@ -174,11 +175,11 @@ public ArrayList getProjectsMetadata() { public JSONObject getProjectMetadata(String projectId) { String metadataPath = "metadata/" + projectId + ".json"; - File metadataFile = new File(metadataPath); + File metadataFile = QuantumBrush.appFile(metadataPath); if (metadataFile.exists()) { try { - return app.loadJSONObject(metadataPath); + return app.loadJSONObject(metadataFile.getAbsolutePath()); } catch (Exception e) { System.err.println("Error loading metadata: " + e.getMessage()); } @@ -190,15 +191,15 @@ public JSONObject getProjectMetadata(String projectId) { // Add a method to load strokes when a project is loaded public boolean loadProject(String projectId) { String projectPath = "project/" + projectId; - File projectDir = new File(projectPath); + File projectDir = QuantumBrush.appFile(projectPath); if (!projectDir.exists()) { return false; } // Try to load current state first (with all effects applied) - File currentImageFile = new File(projectPath + "/current.png"); - File originalImageFile = new File(projectPath + "/original.png"); + File currentImageFile = QuantumBrush.appFile(projectPath + "/current.png"); + File originalImageFile = QuantumBrush.appFile(projectPath + "/original.png"); PImage loadedImage = null; @@ -246,13 +247,13 @@ public boolean deleteProject(String projectId) { try { // Delete project directory and all its contents - File projectDir = new File("project/" + projectId); + File projectDir = QuantumBrush.appFile("project/" + projectId); if (projectDir.exists()) { success &= deleteDirectory(projectDir); } // Delete metadata file - File metadataFile = new File("metadata/" + projectId + ".json"); + File metadataFile = QuantumBrush.appFile("metadata/" + projectId + ".json"); if (metadataFile.exists()) { success &= metadataFile.delete(); } @@ -294,7 +295,7 @@ private boolean deleteDirectory(File directory) { public int cleanupOrphanedMetadata() { int cleanedCount = 0; - File metadataDir = new File("metadata"); + File metadataDir = QuantumBrush.appFile("metadata"); if (!metadataDir.exists()) { return 0; } @@ -308,7 +309,7 @@ public int cleanupOrphanedMetadata() { try { JSONObject metadata = app.loadJSONObject(file.getAbsolutePath()); String projectId = metadata.getString("project_id", ""); - File projectDir = new File("project/" + projectId); + File projectDir = QuantumBrush.appFile("project/" + projectId); // Only delete if project directory truly doesn't exist if (!projectDir.exists()) { diff --git a/src/QuantumBrush.java b/src/QuantumBrush.java index 488dafe..d43cc15 100644 --- a/src/QuantumBrush.java +++ b/src/QuantumBrush.java @@ -6,8 +6,13 @@ import java.awt.event.*; import java.util.*; import java.io.*; +import java.net.URISyntaxException; +import java.nio.file.*; +import java.util.jar.*; public class QuantumBrush extends PApplet { + private static File appRootDirectory = new File(".").getAbsoluteFile(); + // Managers private CanvasManager canvas; private EffectManager effects; @@ -19,6 +24,9 @@ public class QuantumBrush extends PApplet { // UI components private JFrame controlFrame; private JFrame canvasFrame; + private Component processingCanvas; + private JSplitPane workspaceSplit; + private JPanel canvasPanel; private JMenuBar menuBar; private JComboBox effectsDropdown; private JButton createButton; @@ -42,6 +50,8 @@ public class QuantumBrush extends PApplet { private static final int CANVAS_HEIGHT = 600; private static final int MIN_CANVAS_WIDTH = 320; private static final int MIN_CANVAS_HEIGHT = 240; + private static final int CONTROL_PANEL_WIDTH = 420; + private static final int CONTROL_PANEL_MIN_WIDTH = 320; private static final float CANVAS_SCREEN_WIDTH_RATIO = 0.6f; private static final float CANVAS_SCREEN_HEIGHT_RATIO = 0.8f; @@ -55,8 +65,75 @@ public class QuantumBrush extends PApplet { private float strokeWeight = 2.0f; public static void main(String[] args) { + configureAppRoot(); + ensureBundledEffectsAvailable(); PApplet.main("QuantumBrush"); } + + public static File getAppRootDirectory() { + return appRootDirectory; + } + + public static File appFile(String path) { + File file = new File(path); + return file.isAbsolute() ? file : new File(appRootDirectory, path); + } + + private static void configureAppRoot() { + try { + File codeSource = new File( + QuantumBrush.class.getProtectionDomain().getCodeSource().getLocation().toURI() + ); + appRootDirectory = codeSource.isFile() + ? codeSource.getParentFile().getAbsoluteFile() + : new File(".").getAbsoluteFile(); + System.setProperty("user.dir", appRootDirectory.getAbsolutePath()); + } catch (URISyntaxException e) { + appRootDirectory = new File(".").getAbsoluteFile(); + } + } + + private static void ensureBundledEffectsAvailable() { + File effectDir = appFile("effect"); + if (effectDir.isDirectory()) { + return; + } + + try { + File codeSource = new File( + QuantumBrush.class.getProtectionDomain().getCodeSource().getLocation().toURI() + ); + if (!codeSource.isFile()) { + return; + } + + try (JarFile jar = new JarFile(codeSource)) { + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String name = entry.getName(); + if (!name.startsWith("effect/") || name.contains("__pycache__")) { + continue; + } + + File output = appFile(name); + if (entry.isDirectory()) { + output.mkdirs(); + } else { + File parent = output.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + try (InputStream in = jar.getInputStream(entry)) { + Files.copy(in, output.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + } catch (Exception e) { + System.err.println("Could not extract bundled effects: " + e.getMessage()); + } + } public void settings() { size(CANVAS_WIDTH, CANVAS_HEIGHT); @@ -138,9 +215,10 @@ private void cleanupTempFiles() { private void setupUI() { // Get the JFrame from Processing for canvas PSurfaceAWT.SmoothCanvas smoothCanvas = (PSurfaceAWT.SmoothCanvas) ((PSurfaceAWT)surface).getNative(); + processingCanvas = smoothCanvas; canvasFrame = (JFrame) smoothCanvas.getFrame(); - canvasFrame.setTitle("Quantum Brush - Canvas"); - canvasFrame.setMinimumSize(new Dimension(MIN_CANVAS_WIDTH, MIN_CANVAS_HEIGHT)); + canvasFrame.setTitle("Quantum Brush"); + canvasFrame.setMinimumSize(new Dimension(CONTROL_PANEL_MIN_WIDTH + MIN_CANVAS_WIDTH, MIN_CANVAS_HEIGHT)); surface.setResizable(true); canvasFrame.addWindowFocusListener(new WindowAdapter() { @Override @@ -149,11 +227,7 @@ public void windowGainedFocus(WindowEvent e) { } }); - // Position the canvas frame on the right side of the screen - Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); - canvasFrame.setLocation(screenSize.width/2, screenSize.height/4); - - // Create control frame + // Create a single workspace with controls and canvas in the Processing frame. createControlFrame(); // Add keyboard shortcuts for undo/redo and zoom/pan @@ -205,7 +279,7 @@ public boolean dispatchKeyEvent(KeyEvent e) { } }); - canvasFrame.setVisible(true); + fitWorkspaceToCanvasSize(CANVAS_WIDTH, CANVAS_HEIGHT); } // Zoom and Pan methods @@ -273,7 +347,7 @@ public void resizeCanvasToFitImage(PImage image) { zoomLevel = 1.0f; panX = 0; panY = 0; - keepCanvasFrameOnScreen(); + fitWorkspaceToCanvasSize(CANVAS_WIDTH, CANVAS_HEIGHT); return; } @@ -303,7 +377,7 @@ public void resizeCanvasToFitImage(PImage image) { ); surface.setSize(targetWidth, targetHeight); - keepCanvasFrameOnScreen(); + fitWorkspaceToCanvasSize(targetWidth, targetHeight); calculateInitialZoomAndPan(targetWidth, targetHeight); } @@ -318,7 +392,38 @@ private void keepCanvasFrameOnScreen() { int y = Math.max(0, Math.min(frameLocation.y, maxY)); canvasFrame.setLocation(x, y); } - + + private void fitWorkspaceToCanvasSize(int canvasWidth, int canvasHeight) { + if (canvasFrame == null || workspaceSplit == null) { + keepCanvasFrameOnScreen(); + return; + } + + Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); + Insets insets = canvasFrame.getInsets(); + int frameDecorationsWidth = insets.left + insets.right; + int frameDecorationsHeight = insets.top + insets.bottom; + int maxWidth = Math.max( + CONTROL_PANEL_MIN_WIDTH + MIN_CANVAS_WIDTH, + screenSize.width - 80 + ); + int maxHeight = Math.max(MIN_CANVAS_HEIGHT, screenSize.height - 80); + int targetWidth = Math.min( + maxWidth, + CONTROL_PANEL_WIDTH + Math.max(canvasWidth, MIN_CANVAS_WIDTH) + frameDecorationsWidth + 40 + ); + int targetHeight = Math.min( + maxHeight, + Math.max(canvasHeight, MIN_CANVAS_HEIGHT) + frameDecorationsHeight + 80 + ); + + canvasFrame.setSize(targetWidth, targetHeight); + workspaceSplit.setDividerLocation(Math.min(CONTROL_PANEL_WIDTH, targetWidth / 2)); + keepCanvasFrameOnScreen(); + canvasFrame.revalidate(); + canvasFrame.repaint(); + } + // Coordinate transformation methods private PVector screenToImage(float screenX, float screenY) { if (currentImage == null) return new PVector(screenX, screenY); @@ -348,17 +453,13 @@ private boolean isPointInImage(float screenX, float screenY) { } private void createControlFrame() { - // Create main control window - controlFrame = new JFrame("Quantum Brush - Control Panel"); + controlFrame = canvasFrame; + controlFrame.setTitle("Quantum Brush"); Dimension screenSize = Toolkit.getDefaultToolkit().getScreenSize(); - int controlWidth = Math.min(500, (int)(screenSize.width * 0.4)); + int controlWidth = Math.min(CONTROL_PANEL_WIDTH, Math.max(CONTROL_PANEL_MIN_WIDTH, (int)(screenSize.width * 0.35))); int controlHeight = Math.min(700, (int)(screenSize.height * 0.8)); - controlFrame.setSize(controlWidth, controlHeight); controlFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); - // Position the control frame on the left side of the screen - controlFrame.setLocation(screenSize.width/4, screenSize.height/4); - // Set references for UI manager ui.setMainControlFrame(controlFrame); @@ -542,8 +643,28 @@ private void createControlFrame() { mainPanel.add(tabbedPane, BorderLayout.CENTER); mainPanel.add(zoomInfoPanel, BorderLayout.SOUTH); + mainPanel.setMinimumSize(new Dimension(CONTROL_PANEL_MIN_WIDTH, MIN_CANVAS_HEIGHT)); + mainPanel.setPreferredSize(new Dimension(controlWidth, controlHeight)); + + canvasPanel = new JPanel(new BorderLayout()); + canvasPanel.setBorder(BorderFactory.createCompoundBorder( + BorderFactory.createEmptyBorder(10, 0, 10, 10), + BorderFactory.createTitledBorder("Canvas") + )); + canvasPanel.setMinimumSize(new Dimension(MIN_CANVAS_WIDTH, MIN_CANVAS_HEIGHT)); + canvasPanel.setPreferredSize(new Dimension(CANVAS_WIDTH, CANVAS_HEIGHT)); + canvasPanel.add(processingCanvas, BorderLayout.CENTER); - controlFrame.add(mainPanel); + workspaceSplit = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, mainPanel, canvasPanel); + workspaceSplit.setContinuousLayout(true); + workspaceSplit.setResizeWeight(0.0); + workspaceSplit.setDividerLocation(controlWidth); + workspaceSplit.setBorder(BorderFactory.createEmptyBorder()); + + controlFrame.getContentPane().removeAll(); + controlFrame.getContentPane().setLayout(new BorderLayout()); + controlFrame.getContentPane().add(workspaceSplit, BorderLayout.CENTER); + fitWorkspaceToCanvasSize(CANVAS_WIDTH, CANVAS_HEIGHT); // Belt-and-braces: clear the in-memory IQM token if the user closes the // control window (the shutdown hook covers SIGTERM/exit() paths too). @@ -557,6 +678,11 @@ public void windowClosing(WindowEvent e) { }); controlFrame.setVisible(true); + fitWorkspaceToCanvasSize(CANVAS_WIDTH, CANVAS_HEIGHT); + controlFrame.setLocation( + Math.max(0, (screenSize.width - controlFrame.getWidth()) / 2), + Math.max(0, (screenSize.height - controlFrame.getHeight()) / 2) + ); } private void showLiveDebugViewer() { @@ -1054,9 +1180,9 @@ private void initializeProjectHistoryAfterLoad() { private void saveCurrentImageToDisk() { if (projectId != null && currentImage != null) { String projectPath = "project/" + projectId; - File projectDir = new File(projectPath); + File projectDir = appFile(projectPath); if (projectDir.exists()) { - currentImage.save(projectPath + "/current.png"); + currentImage.save(appFile(projectPath + "/current.png").getAbsolutePath()); System.out.println("Saved current image state to disk: project/" + projectId + "/current.png"); } } @@ -1149,16 +1275,41 @@ public void draw() { canvas.draw(zoomLevel, panX, panY); } else { // Show "no image" message + float placeholderX = getVisibleCanvasWidth() / 2.0f; + float placeholderY = getVisibleCanvasHeight() / 2.0f; + fill(200); textAlign(CENTER, CENTER); textSize(18); - text("Load an image or project to begin", width/2, height/2); + text("Load an image or project to begin", placeholderX, placeholderY); fill(150); textSize(14); - text("Use File > New to load an image", width/2, height/2 + 30); - text("Use keyboard shortcuts to zoom: Ctrl/Cmd + Plus/Minus", width/2, height/2 + 50); + text("Use File > New to load an image", placeholderX, placeholderY + 30); + text("Use keyboard shortcuts to zoom: Ctrl/Cmd + Plus/Minus", placeholderX, placeholderY + 50); + } + } + + private int getVisibleCanvasWidth() { + if (canvasPanel != null && canvasPanel.getWidth() > 0) { + Insets insets = canvasPanel.getInsets(); + return Math.max(1, canvasPanel.getWidth() - insets.left - insets.right); + } + if (processingCanvas != null && processingCanvas.getWidth() > 0) { + return processingCanvas.getWidth(); + } + return width; + } + + private int getVisibleCanvasHeight() { + if (canvasPanel != null && canvasPanel.getHeight() > 0) { + Insets insets = canvasPanel.getInsets(); + return Math.max(1, canvasPanel.getHeight() - insets.top - insets.bottom); + } + if (processingCanvas != null && processingCanvas.getHeight() > 0) { + return processingCanvas.getHeight(); } + return height; } // Mouse event handling with coordinate transformation diff --git a/src/StrokeManager.java b/src/StrokeManager.java index 6eabdab..98a8b5f 100644 --- a/src/StrokeManager.java +++ b/src/StrokeManager.java @@ -4,8 +4,11 @@ import java.io.*; import javax.swing.*; import java.awt.*; +import java.awt.image.BufferedImage; import java.nio.file.Files; +import java.nio.charset.StandardCharsets; import java.util.concurrent.*; +import javax.imageio.ImageIO; public class StrokeManager { private QuantumBrush app; @@ -19,6 +22,8 @@ public class StrokeManager { // Map to track which strokes are currently being processed private Map> processingStrokes; + private Map strokeInputImages; + private Map strokeResultImages; // Callback interface for UI updates public interface ProcessingCallback { @@ -32,6 +37,8 @@ public StrokeManager(QuantumBrush app) { this.strokes = new ArrayList<>(); this.executorService = Executors.newFixedThreadPool(3); // Allow up to 3 concurrent processes this.processingStrokes = new ConcurrentHashMap<>(); + this.strokeInputImages = new ConcurrentHashMap<>(); + this.strokeResultImages = new ConcurrentHashMap<>(); this.callbacks = new ArrayList<>(); // Initialize Python command on startup @@ -402,7 +409,7 @@ public String getDescription() { private String loadCustomPythonPath() { try { - File configFile = new File("config/python_path.txt"); + File configFile = QuantumBrush.appFile("config/python_path.txt"); if (configFile.exists()) { return new String(Files.readAllBytes(configFile.toPath())).trim(); } @@ -414,12 +421,12 @@ private String loadCustomPythonPath() { private void saveCustomPythonPath(String path) { try { - File configDir = new File("config"); + File configDir = QuantumBrush.appFile("config"); if (!configDir.exists()) { configDir.mkdirs(); } - File configFile = new File("config/python_path.txt"); + File configFile = QuantumBrush.appFile("config/python_path.txt"); Files.write(configFile.toPath(), path.getBytes()); System.out.println("Saved custom Python path: " + path); } catch (Exception e) { @@ -468,14 +475,14 @@ public String createStroke(Effect effect, Map parameters) { } // Create project directory structure - File projectDir = new File("project/" + projectId); + File projectDir = QuantumBrush.appFile("project/" + projectId); if (!projectDir.exists()) { projectDir.mkdirs(); // Save original image if (app.getCurrentImage() != null) { - String originalPath = projectDir.getPath() + "/original.png"; - app.getCurrentImage().save(originalPath); + File originalPath = new File(projectDir, "original.png"); + app.getCurrentImage().save(originalPath.getAbsolutePath()); } } @@ -486,9 +493,10 @@ public String createStroke(Effect effect, Map parameters) { } // Generate and save JSON instructions - String instructionsPath = strokeDir.getPath() + "/" + strokeId + "_instructions.json"; + String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; + File instructionsFile = QuantumBrush.appFile(instructionsPath); JSONObject instructions = stroke.generateJSON(projectId, app.getHardwareManager().snapshotForStroke()); - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); // DEBUG: Show the final JSON that was saved DebugLogger.log("\n=== FINAL SAVED JSON DEBUG ==="); @@ -506,21 +514,16 @@ public String createStroke(Effect effect, Map parameters) { } DebugLogger.log("=== END FINAL SAVED JSON DEBUG ===\n"); - // Save input image + // Keep the input image in memory. The persistent stroke JSON remains + // lightweight and no per-stroke input PNG is written to disk. if (app.getCurrentImage() != null) { - String inputPath = strokeDir.getPath() + "/" + strokeId + "_input.png"; - app.getCurrentImage().save(inputPath); - - // Update JSON with image paths - instructions = app.loadJSONObject(instructionsPath); + strokeInputImages.put(strokeId, app.getCurrentImage().copy()); + + instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); JSONObject strokeInput = instructions.getJSONObject("stroke_input"); - strokeInput.setString("input_location", inputPath); - strokeInput.setString( - "output_location", - strokeDir.getPath() + "/" + strokeId + "_output.png" - ); + strokeInput.setString("transport", "stdio_png"); instructions.setJSONObject("stroke_input", strokeInput); - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); } // Set current stroke index @@ -536,7 +539,7 @@ public void loadExistingStrokes() { } String projectId = app.getProjectId(); - File strokeDir = new File("project/" + projectId + "/stroke"); + File strokeDir = QuantumBrush.appFile("project/" + projectId + "/stroke"); if (!strokeDir.exists() || !strokeDir.isDirectory()) { return; @@ -626,6 +629,8 @@ public void loadExistingStrokes() { // ✅ FIXED: Extract path data correctly - preserve separate paths // ✅ CRITICAL FIX: Reconstruct SEPARATE paths correctly JSONObject strokeInput = instructions.getJSONObject("stroke_input"); + restorePersistentStrokeImages(strokeId, instructions); + JSONArray pathArray = strokeInput.getJSONArray("path"); JSONArray clicksArray = strokeInput.getJSONArray("clicks"); @@ -846,11 +851,11 @@ public void runStroke(Stroke stroke) { // Define instructionsPath once here to use throughout the method final String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; - File instructionsFile = new File(instructionsPath); + File instructionsFile = QuantumBrush.appFile(instructionsPath); // Check if the stroke is in a failed state and needs cleanup if (instructionsFile.exists()) { - JSONObject instructions = app.loadJSONObject(instructionsPath); + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); String processingStatus = instructions.getString("processing_status", ""); // If the stroke is in a "running" state but not in our processing map, @@ -858,15 +863,15 @@ public void runStroke(Stroke stroke) { if ("running".equals(processingStatus) && !processingStrokes.containsKey(strokeId)) { // Reset the status to allow reprocessing instructions.setString("processing_status", "pending"); - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); } } // Check if Python script exists - File pythonScript = new File("effect/" + folderName + "/" + effectId + ".py"); + File pythonScript = QuantumBrush.appFile("effect/" + folderName + "/" + effectId + ".py"); if (!pythonScript.exists()) { // Try with folder name as fallback - pythonScript = new File("effect/" + folderName + "/" + folderName + ".py"); + pythonScript = QuantumBrush.appFile("effect/" + folderName + "/" + folderName + ".py"); if (!pythonScript.exists()) { JOptionPane.showMessageDialog( null, @@ -879,7 +884,7 @@ public void runStroke(Stroke stroke) { } // Ensure directories exist - File projectDir = new File("project/" + projectId); + File projectDir = QuantumBrush.appFile("project/" + projectId); File strokeDir = new File(projectDir, "stroke"); if (!projectDir.exists()) projectDir.mkdirs(); if (!strokeDir.exists()) strokeDir.mkdirs(); @@ -887,17 +892,17 @@ public void runStroke(Stroke stroke) { // Ensure instructions file exists if (!instructionsFile.exists()) { JSONObject instructions = stroke.generateJSON(projectId, app.getHardwareManager().snapshotForStroke()); - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); } // Update status fields to indicate processing has started - JSONObject instructions = app.loadJSONObject(instructionsPath); + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); instructions.setBoolean("created", true); instructions.setString("effect_received", "null"); instructions.setString("effect_processed", "null"); instructions.setString("effect_success", "null"); instructions.setString("processing_status", "running"); - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); // Create a task to execute the Python script asynchronously Runnable task = () -> { @@ -905,16 +910,16 @@ public void runStroke(Stroke stroke) { try { // Execute Python script with absolute path - success = executeApplyEffectScript(instructionsFile.getAbsolutePath()); + success = executeApplyEffectScriptInMemory(instructionsFile.getAbsolutePath()); // Update status based on result - JSONObject updatedInstructions = app.loadJSONObject(instructionsPath); + JSONObject updatedInstructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); if (success) { updatedInstructions.setString("effect_received", "true"); updatedInstructions.setString("effect_processed", "true"); updatedInstructions.setString("effect_success", "true"); updatedInstructions.setString("processing_status", "completed"); - app.saveJSONObject(updatedInstructions, instructionsPath); + app.saveJSONObject(updatedInstructions, instructionsFile.getAbsolutePath()); // Notify on the Event Dispatch Thread SwingUtilities.invokeLater(() -> { @@ -925,7 +930,7 @@ public void runStroke(Stroke stroke) { updatedInstructions.setString("effect_processed", "true"); updatedInstructions.setString("effect_success", "false"); updatedInstructions.setString("processing_status", "failed"); - app.saveJSONObject(updatedInstructions, instructionsPath); + app.saveJSONObject(updatedInstructions, instructionsFile.getAbsolutePath()); // Notify on the Event Dispatch Thread SwingUtilities.invokeLater(() -> { @@ -938,11 +943,11 @@ public void runStroke(Stroke stroke) { // Update the instructions file to reflect cancellation try { - JSONObject updatedInstructions = app.loadJSONObject(instructionsPath); + JSONObject updatedInstructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); updatedInstructions.setString("effect_success", "false"); updatedInstructions.setString("processing_status", "canceled"); updatedInstructions.setString("error_message", "Process was cancelled by user"); - app.saveJSONObject(updatedInstructions, instructionsPath); + app.saveJSONObject(updatedInstructions, instructionsFile.getAbsolutePath()); } catch (Exception ex) { System.err.println("Error updating instructions file after cancellation: " + ex.getMessage()); } @@ -959,11 +964,11 @@ public void runStroke(Stroke stroke) { e.printStackTrace(); try { - JSONObject updatedInstructions = app.loadJSONObject(instructionsPath); + JSONObject updatedInstructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); updatedInstructions.setString("effect_success", "false"); updatedInstructions.setString("processing_status", "failed"); updatedInstructions.setString("error_message", e.getMessage()); - app.saveJSONObject(updatedInstructions, instructionsPath); + app.saveJSONObject(updatedInstructions, instructionsFile.getAbsolutePath()); } catch (Exception ex) { System.err.println("Error updating instructions file: " + ex.getMessage()); } @@ -1163,10 +1168,10 @@ public String getStrokeProcessingStatus(String strokeId) { try { String projectId = app.getProjectId(); String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; - File instructionsFile = new File(instructionsPath); + File instructionsFile = QuantumBrush.appFile(instructionsPath); if (instructionsFile.exists()) { - JSONObject instructions = app.loadJSONObject(instructionsFile); + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); return instructions.getString("processing_status", "unknown"); } } catch (Exception e) { @@ -1184,14 +1189,14 @@ private boolean executeApplyEffectScript(String instructionsFilePath) throws Int } // Create log directory if it doesn't exist - File logDir = new File("log"); + File logDir = QuantumBrush.appFile("log"); if (!logDir.exists()) { logDir.mkdirs(); } // Create log files for stdout and stderr - File stdoutLog = new File("log/python_stdout.log"); - File stderrLog = new File("log/python_stderr.log"); + File stdoutLog = QuantumBrush.appFile("log/python_stdout.log"); + File stderrLog = QuantumBrush.appFile("log/python_stderr.log"); // Display Python version information ProcessBuilder versionProcessBuilder = new ProcessBuilder(pythonCommand, "--version"); @@ -1231,15 +1236,20 @@ private boolean executeApplyEffectScript(String instructionsFilePath) throws Int versionProcess.waitFor(); // Create command to execute Python script + File instructionsFile = QuantumBrush.appFile(instructionsFilePath); ProcessBuilder processBuilder = new ProcessBuilder( - pythonCommand, "effect/apply_effect.py", instructionsFilePath + pythonCommand, + QuantumBrush.appFile("effect/apply_effect.py").getAbsolutePath(), + instructionsFile.getAbsolutePath() ); + processBuilder.directory(QuantumBrush.getAppRootDirectory()); processBuilder.redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutLog)); processBuilder.redirectError(ProcessBuilder.Redirect.appendTo(stderrLog)); // Pass the IQM token through the subprocess environment when the // user has chosen the IQM backend. The token never lands on disk. app.getHardwareManager().applyToProcessEnv(processBuilder.environment()); + addLocalPythonPackages(processBuilder.environment()); // Execute process process = processBuilder.start(); @@ -1253,11 +1263,11 @@ private boolean executeApplyEffectScript(String instructionsFilePath) throws Int int exitCode = process.exitValue(); // Check the effect_success flag in the JSON file - JSONObject instructions = app.loadJSONObject(instructionsFilePath); + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); String projectId = instructions.getString("project_id", ""); String strokeId = instructions.getString("stroke_id", ""); String outputPath = "project/" + projectId + "/stroke/" + strokeId + "_output.png"; - File outputFile = new File(outputPath); + File outputFile = QuantumBrush.appFile(outputPath); String effectSuccess = instructions.getString("effect_success", "false"); boolean success = exitCode == 0 && "true".equals(effectSuccess) && outputFile.exists(); @@ -1307,6 +1317,530 @@ private boolean executeApplyEffectScript(String instructionsFilePath) throws Int } } + private boolean executeApplyEffectScriptInMemory(String instructionsFilePath) + throws InterruptedException, IOException { + JSONObject instructions = app.loadJSONObject(instructionsFilePath); + String strokeId = instructions.getString("stroke_id", ""); + PImage inputImage = strokeInputImages.get(strokeId); + if (inputImage == null) { + inputImage = app.getCurrentImage(); + } + if (inputImage == null) { + throw new IOException("No input image is available for stroke " + strokeId); + } + + return executeApplyEffectScriptInMemory(instructionsFilePath, inputImage, true); + } + + private boolean executeApplyEffectScriptInMemory( + String instructionsFilePath, + PImage inputImage, + boolean persistResult + ) throws InterruptedException, IOException { + Process process = null; + File stderrLog = QuantumBrush.appFile("log/python_stderr.log"); + + try { + if (pythonCommand == null) { + initializePythonCommand(); + } + + File logDir = QuantumBrush.appFile("log"); + if (!logDir.exists()) { + logDir.mkdirs(); + } + + ProcessBuilder versionProcessBuilder = new ProcessBuilder(pythonCommand, "--version"); + versionProcessBuilder.redirectErrorStream(true); + + Process versionProcess = versionProcessBuilder.start(); + BufferedReader versionReader = new BufferedReader( + new InputStreamReader(versionProcess.getInputStream()) + ); + String versionLine; + while ((versionLine = versionReader.readLine()) != null) { + System.out.println( + "Using Python: " + versionLine + " (from: " + pythonCommand + ")" + ); + + if (versionLine.matches(".*Python 3\\.[0-9](\\..*)?")) { + String minorVersionStr = versionLine.replaceAll( + ".*Python 3\\.([0-9])(\\..*)?", "$1" + ); + try { + int minorVersion = Integer.parseInt(minorVersionStr); + if (minorVersion < 10) { + System.err.println( + "WARNING: Python version is " + versionLine + + " but match-case syntax requires Python 3.10 or higher!" + ); + return false; + } + } catch (NumberFormatException e) { + System.err.println("Could not parse Python version: " + versionLine); + } + } + } + versionProcess.waitFor(); + + JSONObject instructions = app.loadJSONObject(instructionsFilePath); + String strokeId = instructions.getString("stroke_id", ""); + if (inputImage == null) { + throw new IOException("No input image is available for stroke " + strokeId); + } + + ProcessBuilder processBuilder = new ProcessBuilder( + pythonCommand, + QuantumBrush.appFile("effect/apply_effect.py").getAbsolutePath(), + "--stdio" + ); + processBuilder.directory(QuantumBrush.getAppRootDirectory()); + processBuilder.redirectError(ProcessBuilder.Redirect.appendTo(stderrLog)); + + app.getHardwareManager().applyToProcessEnv(processBuilder.environment()); + addLocalPythonPackages(processBuilder.environment()); + + process = processBuilder.start(); + + System.out.println("Starting in-memory effect processing..."); + writeBinaryRequest(process.getOutputStream(), instructions, pImageToPngBytes(inputImage)); + BinaryResponse binaryResponse = readBinaryResponse(process.getInputStream()); + process.waitFor(); + + int exitCode = process.exitValue(); + JSONObject response = JSONObject.parse(binaryResponse.json); + String effectSuccess = response.getString("effect_success", "false"); + boolean success = exitCode == 0 && "true".equals(effectSuccess); + + if (success) { + PImage resultImage = pngBytesToPImage(binaryResponse.imageBytes); + strokeResultImages.put(strokeId, resultImage); + if (persistResult) { + persistStrokeImages(instructionsFilePath, inputImage, resultImage); + } + } else { + System.err.println("Python script execution failed with exit code: " + exitCode); + System.err.println("Effect success flag: " + effectSuccess); + logPythonErrors(stderrLog); + } + + return success; + } catch (InterruptedException e) { + System.err.println("Process execution interrupted"); + + if (process != null && process.isAlive()) { + process.destroyForcibly(); + System.out.println("Forcibly terminated process due to cancellation"); + } + + try { + JSONObject instructions = app.loadJSONObject(instructionsFilePath); + instructions.setString("effect_success", "false"); + instructions.setString("processing_status", "canceled"); + instructions.setString("error_message", "Process was cancelled by user"); + app.saveJSONObject(instructions, instructionsFilePath); + } catch (Exception ex) { + System.err.println("Error updating instructions file after cancellation: " + ex.getMessage()); + } + + throw e; + } + } + + private void addLocalPythonPackages(Map environment) { + File localPackages = QuantumBrush.appFile(".python-packages"); + if (!localPackages.isDirectory()) { + return; + } + + String packagePath = localPackages.getAbsolutePath(); + String currentPath = environment.get("PYTHONPATH"); + if (currentPath == null || currentPath.isEmpty()) { + environment.put("PYTHONPATH", packagePath); + } else if (!Arrays.asList(currentPath.split(File.pathSeparator)).contains(packagePath)) { + environment.put("PYTHONPATH", packagePath + File.pathSeparator + currentPath); + } + } + + private void logPythonErrors(File stderrLog) { + if (!stderrLog.exists()) { + return; + } + + try (BufferedReader reader = new BufferedReader(new FileReader(stderrLog))) { + String line; + System.err.println("Python error log:"); + while ((line = reader.readLine()) != null) { + System.err.println(" " + line); + } + } catch (IOException e) { + System.err.println("Could not read error log: " + e.getMessage()); + } + } + + private static class BinaryResponse { + String json; + byte[] imageBytes; + + BinaryResponse(String json, byte[] imageBytes) { + this.json = json; + this.imageBytes = imageBytes; + } + } + + private void writeBinaryRequest( + OutputStream outputStream, + JSONObject instructions, + byte[] imageBytes + ) throws IOException { + byte[] jsonBytes = instructions.toString().getBytes(StandardCharsets.UTF_8); + DataOutputStream output = new DataOutputStream(new BufferedOutputStream(outputStream)); + output.writeInt(jsonBytes.length); + output.write(jsonBytes); + output.writeInt(imageBytes.length); + output.write(imageBytes); + output.flush(); + output.close(); + } + + private BinaryResponse readBinaryResponse(InputStream inputStream) throws IOException { + DataInputStream input = new DataInputStream(new BufferedInputStream(inputStream)); + int jsonLength = input.readInt(); + if (jsonLength < 0) { + throw new IOException("Python returned an invalid response header."); + } + + byte[] jsonBytes = new byte[jsonLength]; + input.readFully(jsonBytes); + + int imageLength = input.readInt(); + if (imageLength < 0) { + throw new IOException("Python returned an invalid image payload header."); + } + + byte[] imageBytes = new byte[imageLength]; + if (imageLength > 0) { + input.readFully(imageBytes); + } + + return new BinaryResponse(new String(jsonBytes, StandardCharsets.UTF_8), imageBytes); + } + + private byte[] pImageToPngBytes(PImage image) throws IOException { + BufferedImage bufferedImage = new BufferedImage( + image.width, + image.height, + BufferedImage.TYPE_INT_ARGB + ); + image.loadPixels(); + for (int y = 0; y < image.height; y++) { + for (int x = 0; x < image.width; x++) { + bufferedImage.setRGB(x, y, image.pixels[y * image.width + x]); + } + } + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ImageIO.write(bufferedImage, "png", output); + return output.toByteArray(); + } + + private PImage pngBytesToPImage(byte[] imageBytes) throws IOException { + BufferedImage bufferedImage = ImageIO.read(new ByteArrayInputStream(imageBytes)); + if (bufferedImage == null) { + throw new IOException("Python returned an invalid PNG payload."); + } + + PImage image = app.createImage( + bufferedImage.getWidth(), + bufferedImage.getHeight(), + PConstants.ARGB + ); + image.loadPixels(); + for (int y = 0; y < bufferedImage.getHeight(); y++) { + for (int x = 0; x < bufferedImage.getWidth(); x++) { + image.pixels[y * bufferedImage.getWidth() + x] = bufferedImage.getRGB(x, y); + } + } + image.updatePixels(); + return image; + } + + private void persistStrokeImages(String instructionsFilePath, PImage inputImage, PImage resultImage) { + try { + JSONObject instructions = app.loadJSONObject(instructionsFilePath); + JSONObject serialized = new JSONObject(); + + PImage inputPreview = createPreviewImage(inputImage, 360); + serialized.setString("input_preview_png", pImageToBase64Png(inputPreview)); + serialized.setInt("input_preview_width", inputPreview.width); + serialized.setInt("input_preview_height", inputPreview.height); + + JSONObject resultPatch = createResultPatch(resultImage); + serialized.setJSONObject("result_patch", resultPatch); + instructions.setJSONObject("serialized_images", serialized); + + app.saveJSONObject(instructions, instructionsFilePath); + } catch (Exception e) { + System.err.println("Could not persist in-memory stroke result: " + e.getMessage()); + } + } + + private void restorePersistentStrokeImages(String strokeId, JSONObject instructions) { + if (!instructions.hasKey("serialized_images")) { + return; + } + + try { + JSONObject serialized = instructions.getJSONObject("serialized_images"); + String inputPreview = serialized.getString("input_preview_png", ""); + if (!inputPreview.isEmpty()) { + strokeInputImages.put(strokeId, base64PngToPImage(inputPreview)); + } + + if (serialized.hasKey("result_patch")) { + PImage resultImage = restoreResultPatch(serialized.getJSONObject("result_patch")); + if (resultImage != null) { + strokeResultImages.put(strokeId, resultImage); + } + } + } catch (Exception e) { + System.err.println("Could not restore serialized stroke images: " + e.getMessage()); + } + } + + private JSONObject createResultPatch(PImage image) throws IOException { + image.loadPixels(); + + int minX = image.width; + int minY = image.height; + int maxX = -1; + int maxY = -1; + + for (int y = 0; y < image.height; y++) { + for (int x = 0; x < image.width; x++) { + int pixel = image.pixels[y * image.width + x]; + int alpha = (pixel >>> 24) & 0xFF; + if (alpha != 0) { + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + } + } + } + + JSONObject patch = new JSONObject(); + patch.setInt("canvas_width", image.width); + patch.setInt("canvas_height", image.height); + + if (maxX < minX || maxY < minY) { + patch.setBoolean("empty", true); + patch.setInt("x", 0); + patch.setInt("y", 0); + patch.setInt("width", 0); + patch.setInt("height", 0); + patch.setString("png", ""); + return patch; + } + + int patchWidth = maxX - minX + 1; + int patchHeight = maxY - minY + 1; + PImage crop = app.createImage(patchWidth, patchHeight, PConstants.ARGB); + crop.loadPixels(); + for (int y = 0; y < patchHeight; y++) { + for (int x = 0; x < patchWidth; x++) { + crop.pixels[y * patchWidth + x] = image.pixels[(minY + y) * image.width + minX + x]; + } + } + crop.updatePixels(); + + patch.setBoolean("empty", false); + patch.setInt("x", minX); + patch.setInt("y", minY); + patch.setInt("width", patchWidth); + patch.setInt("height", patchHeight); + patch.setString("png", pImageToBase64Png(crop)); + return patch; + } + + private PImage restoreResultPatch(JSONObject patch) throws IOException { + int canvasWidth = patch.getInt("canvas_width", 0); + int canvasHeight = patch.getInt("canvas_height", 0); + if (canvasWidth <= 0 || canvasHeight <= 0) { + return null; + } + + PImage image = app.createImage(canvasWidth, canvasHeight, PConstants.ARGB); + image.loadPixels(); + Arrays.fill(image.pixels, 0); + + if (!patch.getBoolean("empty", false)) { + PImage crop = base64PngToPImage(patch.getString("png", "")); + int offsetX = patch.getInt("x", 0); + int offsetY = patch.getInt("y", 0); + crop.loadPixels(); + for (int y = 0; y < crop.height; y++) { + for (int x = 0; x < crop.width; x++) { + int targetX = offsetX + x; + int targetY = offsetY + y; + if (targetX >= 0 && targetX < canvasWidth && targetY >= 0 && targetY < canvasHeight) { + image.pixels[targetY * canvasWidth + targetX] = crop.pixels[y * crop.width + x]; + } + } + } + } + + image.updatePixels(); + return image; + } + + private PImage createPreviewImage(PImage image, int maxSize) { + int width = image.width; + int height = image.height; + float scale = Math.min(1.0f, maxSize / (float)Math.max(width, height)); + int previewWidth = Math.max(1, Math.round(width * scale)); + int previewHeight = Math.max(1, Math.round(height * scale)); + + if (previewWidth == width && previewHeight == height) { + return image.copy(); + } + + PImage preview = image.copy(); + preview.resize(previewWidth, previewHeight); + return preview; + } + + private String pImageToBase64Png(PImage image) throws IOException { + return Base64.getEncoder().encodeToString(pImageToPngBytes(image)); + } + + private PImage base64PngToPImage(String encoded) throws IOException { + if (encoded == null || encoded.isEmpty()) { + throw new IOException("Missing encoded PNG data."); + } + return pngBytesToPImage(Base64.getDecoder().decode(encoded)); + } + + private PImage processStrokeImageInMemory(String instructionsFilePath, PImage inputImage) + throws InterruptedException, IOException { + boolean success = executeApplyEffectScriptInMemory(instructionsFilePath, inputImage, true); + if (!success) { + return null; + } + + JSONObject instructions = app.loadJSONObject(instructionsFilePath); + return strokeResultImages.get(instructions.getString("stroke_id", "")); + } + + private PImage resolveInputImageForStroke(String strokeId) + throws InterruptedException, IOException { + PImage inputImage = strokeInputImages.get(strokeId); + if (inputImage != null) { + return inputImage.copy(); + } + + inputImage = reconstructImageBeforeStroke(strokeId); + if (inputImage != null) { + strokeInputImages.put(strokeId, inputImage.copy()); + return inputImage; + } + + PImage currentImage = app.getCurrentImage(); + return currentImage != null ? currentImage.copy() : null; + } + + private PImage reconstructImageBeforeStroke(String targetStrokeId) + throws InterruptedException, IOException { + String projectId = app.getProjectId(); + if (projectId == null || targetStrokeId == null || targetStrokeId.isEmpty()) { + return null; + } + + PImage baseImage = app.loadImage( + QuantumBrush.appFile("project/" + projectId + "/original.png").getAbsolutePath() + ); + if (baseImage == null) { + return null; + } + + ArrayList orderedStrokes = new ArrayList<>(strokes); + orderedStrokes.sort((a, b) -> Long.compare(strokeTimestamp(a.getId()), strokeTimestamp(b.getId()))); + + PImage imageBeforeTarget = baseImage.copy(); + for (Stroke stroke : orderedStrokes) { + String strokeId = stroke.getId(); + if (strokeId.equals(targetStrokeId)) { + return imageBeforeTarget; + } + + String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; + File instructionsFile = QuantumBrush.appFile(instructionsPath); + if (!instructionsFile.exists()) { + continue; + } + + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); + boolean appliedOrCompleted = + "true".equals(instructions.getString("effect_success", "false")) || + "completed".equals(instructions.getString("processing_status", "")); + if (!appliedOrCompleted) { + continue; + } + + PImage effectImage = strokeResultImages.get(strokeId); + if (effectImage == null) { + restorePersistentStrokeImages(strokeId, instructions); + effectImage = strokeResultImages.get(strokeId); + } + if (effectImage == null) { + String outputPath = "project/" + projectId + "/stroke/" + strokeId + "_output.png"; + effectImage = app.loadImage(QuantumBrush.appFile(outputPath).getAbsolutePath()); + } + if (effectImage == null) { + strokeInputImages.put(strokeId, imageBeforeTarget.copy()); + effectImage = processStrokeImageInMemory(instructionsFile.getAbsolutePath(), imageBeforeTarget.copy()); + } + if (effectImage != null) { + imageBeforeTarget = blendEffectOntoImage(imageBeforeTarget, effectImage); + } + } + + return null; + } + + private PImage blendEffectOntoImage(PImage baseImage, PImage effectImage) { + PImage resultImage = app.createImage(baseImage.width, baseImage.height, PConstants.ARGB); + resultImage.copy( + baseImage, + 0, 0, baseImage.width, baseImage.height, + 0, 0, resultImage.width, resultImage.height + ); + resultImage.blend( + effectImage, + 0, 0, effectImage.width, effectImage.height, + 0, 0, resultImage.width, resultImage.height, + PConstants.BLEND + ); + return resultImage; + } + + private long strokeTimestamp(String strokeId) { + if (strokeId == null) { + return 0L; + } + + int underscore = strokeId.indexOf('_'); + if (underscore < 0 || underscore + 1 >= strokeId.length()) { + return 0L; + } + + try { + return Long.parseLong(strokeId.substring(underscore + 1)); + } catch (NumberFormatException e) { + return 0L; + } + } + /** * FIXED: Apply effect to canvas by properly layering on top of current image * This method now correctly preserves previous effects and creates proper undo states @@ -1321,7 +1855,7 @@ public boolean applyEffectToCanvas(String strokeId) { // Check if the effect was successful String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; - File instructionsFile = new File(instructionsPath); + File instructionsFile = QuantumBrush.appFile(instructionsPath); if (!instructionsFile.exists()) { JOptionPane.showMessageDialog( @@ -1333,7 +1867,7 @@ public boolean applyEffectToCanvas(String strokeId) { return false; } - JSONObject instructions = app.loadJSONObject(instructionsPath); + JSONObject instructions = app.loadJSONObject(instructionsFile.getAbsolutePath()); String effectSuccess = instructions.getString("effect_success", "false"); if (!"true".equals(effectSuccess)) { @@ -1346,15 +1880,36 @@ public boolean applyEffectToCanvas(String strokeId) { return false; } - // Get the effect output image (just the effect, not layered) - String outputPath = "project/" + projectId + "/stroke/" + strokeId + "_output.png"; - PImage effectImage = app.loadImage(outputPath); + // Get the effect output image (just the effect, not layered). New + // strokes keep the full-resolution result in memory instead of writing + // a per-stroke output PNG. Legacy projects can still load output files. + PImage effectImage = strokeResultImages.get(strokeId); + if (effectImage == null) { + restorePersistentStrokeImages(strokeId, instructions); + effectImage = strokeResultImages.get(strokeId); + } + if (effectImage == null) { + String outputPath = "project/" + projectId + "/stroke/" + strokeId + "_output.png"; + effectImage = app.loadImage(QuantumBrush.appFile(outputPath).getAbsolutePath()); + } + if (effectImage == null) { + try { + PImage inputImage = resolveInputImageForStroke(strokeId); + if (inputImage != null) { + effectImage = processStrokeImageInMemory(instructionsFile.getAbsolutePath(), inputImage); + } + } catch (Exception e) { + System.err.println( + "Could not regenerate effect output for stroke " + strokeId + ": " + e.getMessage() + ); + } + } if (effectImage == null) { - System.err.println("Failed to load effect output image: " + outputPath); + System.err.println("No processed effect output is available for stroke: " + strokeId); JOptionPane.showMessageDialog( null, - "Failed to load effect output image.", + "No processed effect output is available. Please re-run the effect.", "Error", JOptionPane.ERROR_MESSAGE ); @@ -1409,16 +1964,6 @@ public boolean applyEffectToCanvas(String strokeId) { // Update project metadata to reflect the change app.getFileManager().updateProjectMetadata(projectId); - // IMPORTANT: Save the current state to disk - if (projectId != null) { - String projectPath = "project/" + projectId; - File projectDir = new File(projectPath); - if (projectDir.exists()) { - resultImage.save(projectPath + "/current.png"); - System.out.println("Saved current state after applying effect"); - } - } - // Clear drawing paths (but don't save this as a project state) app.getCanvasManager().clearPaths(); @@ -1446,12 +1991,14 @@ public boolean deleteStroke(String strokeId) { // Remove from strokes list strokes.removeIf(stroke -> stroke.getId().equals(strokeId)); + strokeInputImages.remove(strokeId); + strokeResultImages.remove(strokeId); // Delete stroke files String strokeDir = "project/" + projectId + "/stroke/"; - File instructionsFile = new File(strokeDir + strokeId + "_instructions.json"); - File inputFile = new File(strokeDir + strokeId + "_input.png"); - File outputFile = new File(strokeDir + strokeId + "_output.png"); + File instructionsFile = QuantumBrush.appFile(strokeDir + strokeId + "_instructions.json"); + File inputFile = QuantumBrush.appFile(strokeDir + strokeId + "_input.png"); + File outputFile = QuantumBrush.appFile(strokeDir + strokeId + "_output.png"); boolean success = true; if (instructionsFile.exists()) { @@ -1493,11 +2040,31 @@ public Stroke getStroke(int index) { public boolean hasStrokes() { return !strokes.isEmpty(); } + + public boolean hasInMemoryInput(String strokeId) { + return strokeInputImages.containsKey(strokeId); + } + + public boolean hasInMemoryResult(String strokeId) { + return strokeResultImages.containsKey(strokeId); + } + + public PImage getInMemoryInput(String strokeId) { + PImage image = strokeInputImages.get(strokeId); + return image != null ? image.copy() : null; + } + + public PImage getInMemoryResult(String strokeId) { + PImage image = strokeResultImages.get(strokeId); + return image != null ? image.copy() : null; + } public void clearStrokes() { strokes.clear(); currentStrokeIndex = -1; processingStrokes.clear(); + strokeInputImages.clear(); + strokeResultImages.clear(); } /** @@ -1547,13 +2114,14 @@ public boolean updateStroke(String strokeId, Map newParameters) String projectId = app.getProjectId(); JSONObject instructions = strokeToUpdate.generateJSON(projectId, app.getHardwareManager().snapshotForStroke()); String instructionsPath = "project/" + projectId + "/stroke/" + strokeId + "_instructions.json"; + File instructionsFile = QuantumBrush.appFile(instructionsPath); // Mark the stroke as pending again, as its parameters have changed. // This is important so it can be re-run. instructions.setString("processing_status", "pending"); instructions.setString("effect_success", "null"); // Reset success flag - app.saveJSONObject(instructions, instructionsPath); + app.saveJSONObject(instructions, instructionsFile.getAbsolutePath()); System.out.println("Stroke " + strokeId + " updated successfully. Its status has been reset to 'pending'."); return true; diff --git a/src/UIManager.java b/src/UIManager.java index 6868949..4935f0b 100644 --- a/src/UIManager.java +++ b/src/UIManager.java @@ -530,10 +530,10 @@ private void updateStrokeDetails(JPanel detailsPanel, StrokeManager.Stroke strok String projectIdForCost = app.getProjectId(); if (projectIdForCost != null) { String instrPath = "project/" + projectIdForCost + "/stroke/" + stroke.getId() + "_instructions.json"; - java.io.File instrFile = new java.io.File(instrPath); + java.io.File instrFile = QuantumBrush.appFile(instrPath); if (instrFile.exists()) { try { - JSONObject instr = app.loadJSONObject(instrPath); + JSONObject instr = app.loadJSONObject(instrFile.getAbsolutePath()); if (instr.hasKey("cost_estimate_qpu_seconds")) { float cost = instr.getFloat("cost_estimate_qpu_seconds", 0.0f); JSONObject hwBlock = instr.hasKey("hardware") ? instr.getJSONObject("hardware") : null; @@ -573,11 +573,23 @@ private void updateStrokeDetails(JPanel detailsPanel, StrokeManager.Stroke strok if (projectId != null) { // Input Image String inputPath = "project/" + projectId + "/stroke/" + stroke.getId() + "_input.png"; - addImageWithOverlay(imagesPanel, inputPath, "Input", stroke); + if (QuantumBrush.appFile(inputPath).exists()) { + addImageWithOverlay(imagesPanel, QuantumBrush.appFile(inputPath).getAbsolutePath(), "Input", stroke); + } else if (strokeManager.hasInMemoryInput(stroke.getId())) { + addPImageWithOverlay(imagesPanel, strokeManager.getInMemoryInput(stroke.getId()), "Input", stroke); + } else { + addImagePlaceholder(imagesPanel, "Input will be captured when run", "Input"); + } // Output Image String outputPath = "project/" + projectId + "/stroke/" + stroke.getId() + "_output.png"; - addImageWithOverlay(imagesPanel, outputPath, "Output", null); + if (QuantumBrush.appFile(outputPath).exists()) { + addImageWithOverlay(imagesPanel, QuantumBrush.appFile(outputPath).getAbsolutePath(), "Output", null); + } else if (strokeManager.hasInMemoryResult(stroke.getId())) { + addPImageWithOverlay(imagesPanel, strokeManager.getInMemoryResult(stroke.getId()), "Output", null); + } else { + addImagePlaceholder(imagesPanel, "Not processed yet", "Output"); + } } detailsPanel.add(imagesPanel); detailsPanel.add(Box.createVerticalStrut(10)); @@ -649,12 +661,61 @@ private void addImageWithOverlay(JPanel container, String imagePath, String titl } } + private void addPImageWithOverlay(JPanel container, PImage image, String title, StrokeManager.Stroke stroke) { + if (image == null) { + addImagePlaceholder(container, "Image not available", title); + return; + } + + BufferedImage bImg = pImageToBufferedImage(image); + if (stroke != null) { + bImg = createImageWithExactPathOverlay(bImg, stroke); + } + + addBufferedImagePreview(container, bImg, title); + } + + private BufferedImage pImageToBufferedImage(PImage image) { + BufferedImage bufferedImage = new BufferedImage( + image.width, + image.height, + BufferedImage.TYPE_INT_ARGB + ); + image.loadPixels(); + for (int y = 0; y < image.height; y++) { + for (int x = 0; x < image.width; x++) { + bufferedImage.setRGB(x, y, image.pixels[y * image.width + x]); + } + } + return bufferedImage; + } + + private void addBufferedImagePreview(JPanel container, BufferedImage bImg, String title) { + int maxSize = 300; + float scale = Math.min(1, (float)maxSize / Math.max(bImg.getWidth(), bImg.getHeight())); + int displayWidth = (int)(bImg.getWidth() * scale); + int displayHeight = (int)(bImg.getHeight() * scale); + + ImageIcon icon = new ImageIcon(bImg.getScaledInstance(displayWidth, displayHeight, Image.SCALE_SMOOTH)); + JLabel label = new JLabel(icon); + label.setBorder(BorderFactory.createTitledBorder(title)); + container.add(label); + } + private BufferedImage createImageWithExactPathOverlay(BufferedImage baseImage, StrokeManager.Stroke stroke) { BufferedImage overlayImage = new BufferedImage(baseImage.getWidth(), baseImage.getHeight(), BufferedImage.TYPE_INT_ARGB); Graphics2D g2d = overlayImage.createGraphics(); g2d.drawImage(baseImage, 0, 0, null); g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + float scaleX = 1.0f; + float scaleY = 1.0f; + PImage currentImage = app.getCurrentImage(); + if (currentImage != null && currentImage.width > 0 && currentImage.height > 0) { + scaleX = (float) baseImage.getWidth() / currentImage.width; + scaleY = (float) baseImage.getHeight() / currentImage.height; + } + for (Path path : stroke.getPaths()) { // Draw path line g2d.setColor(Color.RED); @@ -663,15 +724,22 @@ private BufferedImage createImageWithExactPathOverlay(BufferedImage baseImage, S for (int i = 0; i < points.size() - 1; i++) { PVector p1 = points.get(i); PVector p2 = points.get(i + 1); - g2d.drawLine((int)p1.x, (int)p1.y, (int)p2.x, (int)p2.y); + g2d.drawLine( + Math.round(p1.x * scaleX), + Math.round(p1.y * scaleY), + Math.round(p2.x * scaleX), + Math.round(p2.y * scaleY) + ); } // Draw click point PVector clickPoint = path.getClickPoint(); if (clickPoint != null) { + int clickX = Math.round(clickPoint.x * scaleX); + int clickY = Math.round(clickPoint.y * scaleY); g2d.setColor(Color.YELLOW); - g2d.fillOval((int)clickPoint.x - 5, (int)clickPoint.y - 5, 10, 10); + g2d.fillOval(clickX - 5, clickY - 5, 10, 10); g2d.setColor(Color.BLACK); - g2d.drawOval((int)clickPoint.x - 5, (int)clickPoint.y - 5, 10, 10); + g2d.drawOval(clickX - 5, clickY - 5, 10, 10); } } g2d.dispose();