Feat/20230701 pipe process

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ["2.7", "3.7", "3.9"]
+        python-version: ["3.7", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
@@ -24,8 +24,9 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip install pytest
       - name: Run unittests
         env:
           PYTHONPATH: ${{ github.workspace }}/src      
         run: |
-          python -m unittest discover -s ./tests/src/win
+          python -m pytest ./tests -v

.gitignore

@@ -4,3 +4,9 @@ pipenv/
 src/puzzle.egg-info/
 tests/jobs/
 tests/_old/
+tests/tests_data/jobs/*
+tests/tests_data/jobs_local/*
+src/puzzle2/plugins/deadline_old/*
+docs/*
+src/puzzle2.egg-info/*
+.coverage

README.md

@@ -1,2 +1,48 @@
-document  
-https://hat27.github.io/puzzle2/
+Puzzle2
+
+Puzzle2 is a lightweight task runner/orchestrator for DCC applications (Maya, MotionBuilder, Houdini, etc.). It standardizes batch execution, logging, environment handoff, and integrates with Deadline.
+
+Docs
+- Website: https://hat27.github.io/puzzle2/
+
+Quick install
+- From source (editable):
+	- pip install -e .
+
+Core pieces
+- puzzle2.Puzzle: Orchestrates steps and tasks with structured results.
+- puzzle2.run_process: Resolves and launches DCC commands in batch/headless mode and writes artifacts under a job directory.
+- puzzle2.PzLog: Scoped logger with JSON/YAML template support, handler-level control, and a Details API for structured results.
+- puzzle2.plugins.deadline: Thin client for building Job/Plugin info and submitting via deadlinecommand.
+
+Running tests (opt-in DCC/Deadline)
+- DCC and Deadline tests are opt-in and gated by an environment variable. Only the following values enable them: 1, true, yes, on (case-insensitive). Any other value or unset disables them.
+	- Example (PowerShell):
+		- $env:PUZZLE_RUN_DCC_TESTS = "1"
+- You can control which app/version pairs are used via environment variables:
+	- PUZZLE_TEST_MATRIX_JSON: JSON string overriding the entire matrix
+		- Example:
+			- $env:PUZZLE_TEST_MATRIX_JSON = '{"mayapy":["2024"],"houdini":["20.5.654"]}'
+	- PUZZLE_TEST_APPS: Semicolon-separated pairs app:version (no spaces)
+		- Example:
+			- $env:PUZZLE_TEST_APPS = "mayapy:2024;houdini:20.5.654"
+
+Deadline client usage
+- Minimal example to submit a job that executes tests/tasks on a Worker:
+- Python:
+	- from puzzle2.plugins.deadline import client
+	- res = client.submit(
+			app="mayapy",
+			job_name="Puzzle2 Example",
+			version="2024",
+			module_path="<path to tests_data or your tasks>",
+			sys_path="<additional sys.path entry>",
+			task_path="<task_set.json or .yml converted to JSON>",
+			data_path="<data.json>",
+			result_path="<shared results.json>",
+		)
+	- print(res.completed.stdout)
+
+Notes
+- CI should keep PUZZLE_RUN_DCC_TESTS disabled by default to avoid external dependency on installed DCCs and a Deadline farm. Opt-in per job when infrastructure is available.
+- PzLog selects a YAML or JSON template automatically depending on availability; handlers are scoped to avoid global logging side effects.

setup.cfg

@@ -3,18 +3,20 @@ name = puzzle2
 description = simple job framework
 author = Goh.Hattori, Narimitu.Ozaki
 license = MIT
-version = 1.0.5
+version = 2.0.0
 platform = linux, win32
 classifiers = 
-    Programming Language :: Python :: 2.7
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
 
 [options]    
 packages = puzzle2
 install_requires = 
     pyyaml
-python_requires = >=2.7
+python_requires = >=3.7
 package_dir = 
     =src
 

src/puzzle2/Puzzle.py

@@ -8,11 +8,11 @@
 import datetime
 import traceback
 import importlib
-import subprocess
 
 from . import PzLog
 from . import pz_env as pz_env
 from . import pz_config as pz_config
+from . import run_process as run_process
 from .PzTask import PzTask
 
 try:
@@ -68,6 +68,7 @@ def __init__(self, name="puzzle", **kwargs):
             if task_directory not in sys.path:
                 sys.path.append(task_directory)
 
+
     def execute_step(self, tasks, data, common, step):
         """
         when data is list, same tasks run for each.
@@ -96,6 +97,7 @@ def execute_step(self, tasks, data, common, step):
             "common" is special keyword.
             we can use them everywhere
             """
+
             temp_common = copy.deepcopy(common)
             temp_common.update(data)
             data = temp_common
@@ -138,7 +140,7 @@ def execute_task(self, task, data):
         task.setdefault("name", module_path.split(".")[-1])
 
         # initialize details log
-        self.logger.details.set_name(task["name"])
+        task_index, index_name = self.logger.details.set_name(task["name"])
         self.logger.details.set_header(0, "successed: {}".format(task["name"]))
         if "comment" in task and task["comment"] != "":
             self.logger.details.add_detail(task["comment"])
@@ -172,14 +174,14 @@ def execute_task(self, task, data):
             response = {"return_code": 0}
 
         self.logger.details.update_code(response["return_code"])
+
         return response
 
     def execute(self, task, data, module):
         task = PzTask(module=module,
-                        task=task,
-                        data=data,
-                        context=self.context)
-
+                      task=task,
+                      data=data,
+                      context=self.context)
         response = task.execute()  # {"return_code": A, "data_globals": B}
 
         if "update_context" in response:
@@ -269,18 +271,42 @@ def _append_sys_path(path):
                 self.logger.debug("break: {}".format(step_name))
                 break
 
-            [_append_sys_path(l) for l in step.get("sys_path", "").split(";") if l != ""]
-
-            now = datetime.datetime.now()
             data_set.setdefault(step_name, {})
-            self.logger.debug("- {} start - {}".format(step_name, step.get("comment", "")))
-
-            self.execute_step(tasks=step["tasks"],
-                          data=data_set[step_name],
-                          common=common,
-                          step=step_name)
-
-            self.logger.info("- {} takes: {}-\n".format(step_name, datetime.datetime.now() - now))
+            if "pipe" in step:
+                pipe_data = copy.deepcopy(step["pipe"])
+                pipe_data["task_set"] = step["tasks"]
+                pipe_data["data_set"] = data_set
+                pipe_data["context"] = copy.deepcopy(self.context)
+                pipe_data["close_app"] = step.get("close_app", True)
+                if "sys_path" in step and step["sys_path"]:
+                    pipe_data["sys_path"] = step["sys_path"]
+                response = run_process.run_process(**pipe_data)
+                command, job_directory = response
+                result_path = "{}/results.json".format(job_directory)
+
+                if command:
+                    if os.path.exists(result_path):
+                        info, data = pz_config.read(result_path)
+                        self.logger.details.add_data_set(data["headers"], 
+                                                         data["results"], 
+                                                         location=step["pipe"])
+
+                        for key, value in data["context"].items():
+                            if key.startswith("_") or key == "logger":
+                                continue
+                            self.context[key] = value
+
+            else:
+                [_append_sys_path(l) for l in step.get("sys_path", "").split(";") if l != ""]
+                now = datetime.datetime.now()
+                self.logger.debug("- {} start - {}".format(step_name, step.get("comment", "")))
+
+                self.execute_step(tasks=step["tasks"],
+                                  data=data_set[step_name],
+                                  common=common,
+                                  step=step_name)
+
+                self.logger.info("- {} takes: {}-\n".format(step_name, datetime.datetime.now() - now))
 
         if steps[-1]["step"] == "closure":
             self.break_ = False