From c07e55fac17904a805a7b08e2fe0f9dde2259081 Mon Sep 17 00:00:00 2001 From: saimanikant Date: Thu, 28 Nov 2024 13:03:40 +0000 Subject: [PATCH 1/3] Add download files to example Python Large Output Files --- .../download_output_file.py | 93 +++++++++++++++++++ examples/python_large_output/project_setup.py | 12 ++- tests/test_examples.py | 21 +++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 examples/python_large_output/download_output_file.py diff --git a/examples/python_large_output/download_output_file.py b/examples/python_large_output/download_output_file.py new file mode 100644 index 000000000..24953afd4 --- /dev/null +++ b/examples/python_large_output/download_output_file.py @@ -0,0 +1,93 @@ +# Copyright (C) 2022 - 2024 ANSYS, Inc. and/or its affiliates. +# SPDX-License-Identifier: MIT +# +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Example to query resources from a project. + +- Query values from evaluated jobs, computing some simple statistics on parameter values. +- Download files from the project + +""" +import argparse +import logging +import os +import time + +from ansys.hps.client import Client, HPSError +from ansys.hps.client.jms import JmsApi, ProjectApi + +log = logging.getLogger(__name__) + + +def download_files(client, project_name): + """Download files.""" + out_path = os.path.join(os.path.dirname(__file__), "downloads") + log.info(f"Downloading files to {out_path}") + + jms_api = JmsApi(client) + project = jms_api.get_project_by_name(name=project_name) + project = jms_api.get_project(id=project.id) + + log.info(f"Project id: {project.id}") + project_api = ProjectApi(client, project.id) + + jobs = project_api.get_jobs(eval_status="evaluated", fields=["id", "values", "elapsed_time"]) + log.info(f"# evaluated jobs: {len(jobs)}") + num = len(jobs) + + log.info( + f"=== Example 1: Downloading output files of {num} jobs using ProjectApi.download_file()" + ) + for job in jobs[0:num]: + log.info(f"Job {job.id}") + for task in project_api.get_tasks(job_id=job.id): + log.info(f"Task {task.id}") + files = project_api.get_files(id=task.output_file_ids) + for f in files: + fpath = os.path.join(out_path, f"task_{task.id}") + log.info(f"Download output file {f.evaluation_path} to {fpath}") + start = time.process_time() + project_api.download_file(file=f, target_path=fpath) + log.info(f"Time taken to download output file: {(time.time() - start):.2f} seconds" + ) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("-n", "--name", type=str, default="Python Large Output Files") + parser.add_argument("-U", "--url", default="https://127.0.0.1:8443/hps") + parser.add_argument("-u", "--username", default="repuser") + parser.add_argument("-p", "--password", default="repuser") + args = parser.parse_args() + + logger = logging.getLogger() + logging.basicConfig(format="%(message)s", level=logging.DEBUG) + + try: + log.info("Connect to HPC Platform Services") + client = Client(url=args.url, username=args.username, password=args.password) + log.info(f"HPS URL: {client.url}") + + download_files(client=client, project_name=args.name) + + except HPSError as e: + log.error(str(e)) \ No newline at end of file diff --git a/examples/python_large_output/project_setup.py b/examples/python_large_output/project_setup.py index de1b2810d..c79ec8446 100644 --- a/examples/python_large_output/project_setup.py +++ b/examples/python_large_output/project_setup.py @@ -46,12 +46,17 @@ log = logging.getLogger(__name__) -def main(client, use_exec_script, python_version=None) -> Project: +def create_project(client, name, use_exec_script, python_version=None) -> Project: """ Create project that runs a Python script to generate a large output file. + + After creating the project, 7 job definitions are created each meant to create + a Large download file of variable size in GB. + + Download files from the project """ log.debug("=== Project") - proj = Project(name="Python Large Output Files", priority=1, active=True) + proj = Project(name=name, priority=1, active=True) jms_api = JmsApi(client) proj = jms_api.create_project(proj, replace=True) project_api = ProjectApi(client, proj.id) @@ -169,6 +174,7 @@ def main(client, use_exec_script, python_version=None) -> Project: if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("-n", "--name", type=str, default="Python Large Output Files") parser.add_argument("-U", "--url", default="https://127.0.0.1:8443/hps") parser.add_argument("-u", "--username", default="repuser") parser.add_argument("-p", "--password", default="repuser") @@ -182,6 +188,6 @@ def main(client, use_exec_script, python_version=None) -> Project: client = Client(url=args.url, username=args.username, password=args.password) try: - main(client, use_exec_script=args.use_exec_script, python_version=args.python_version) + create_project(client, name=args.name, use_exec_script=args.use_exec_script, python_version=args.python_version) except HPSError as e: log.error(str(e)) diff --git a/tests/test_examples.py b/tests/test_examples.py index 10ecc5955..5e82a0cfa 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -274,6 +274,27 @@ def test_cfx_static_mixer(client): jms_api.delete_project(project) +def test_python_large_output(client): + + from examples.python_large_output.project_setup import create_project + + project = create_project( + client, name="Python Large Output Files test", use_exec_script=True, python_version=3.10 + ) + assert project is not None + + jms_api = JmsApi(client) + project_api = ProjectApi(client, project.id) + + assert len(project_api.get_jobs()) == 7 + jms_api.get_project(id=project.id).name == "Python Large Output Files test" + + job = project_api.get_jobs()[0] + assert job.name == "Job 1 GB" + + jms_api.delete_project(project) + + def test_python_multi_steps(client): from examples.python_multi_process_step.project_setup import main as create_project From 3a192aa1beb799d5947fbd1de56f832628c478bf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:47:03 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/python_large_output/download_output_file.py | 10 +++++----- examples/python_large_output/project_setup.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/python_large_output/download_output_file.py b/examples/python_large_output/download_output_file.py index 24953afd4..13c9c1b61 100644 --- a/examples/python_large_output/download_output_file.py +++ b/examples/python_large_output/download_output_file.py @@ -44,7 +44,7 @@ def download_files(client, project_name): log.info(f"Downloading files to {out_path}") jms_api = JmsApi(client) - project = jms_api.get_project_by_name(name=project_name) + project = jms_api.get_project_by_name(name=project_name) project = jms_api.get_project(id=project.id) log.info(f"Project id: {project.id}") @@ -66,9 +66,9 @@ def download_files(client, project_name): fpath = os.path.join(out_path, f"task_{task.id}") log.info(f"Download output file {f.evaluation_path} to {fpath}") start = time.process_time() - project_api.download_file(file=f, target_path=fpath) - log.info(f"Time taken to download output file: {(time.time() - start):.2f} seconds" - ) + project_api.download_file(file=f, target_path=fpath) + log.info(f"Time taken to download output file: {(time.time() - start):.2f} seconds") + if __name__ == "__main__": @@ -90,4 +90,4 @@ def download_files(client, project_name): download_files(client=client, project_name=args.name) except HPSError as e: - log.error(str(e)) \ No newline at end of file + log.error(str(e)) diff --git a/examples/python_large_output/project_setup.py b/examples/python_large_output/project_setup.py index c79ec8446..a39baa6fb 100644 --- a/examples/python_large_output/project_setup.py +++ b/examples/python_large_output/project_setup.py @@ -188,6 +188,11 @@ def create_project(client, name, use_exec_script, python_version=None) -> Projec client = Client(url=args.url, username=args.username, password=args.password) try: - create_project(client, name=args.name, use_exec_script=args.use_exec_script, python_version=args.python_version) + create_project( + client, + name=args.name, + use_exec_script=args.use_exec_script, + python_version=args.python_version, + ) except HPSError as e: log.error(str(e)) From e7b7b773802ea8bb2e2caaa86a60bf537b7e638f Mon Sep 17 00:00:00 2001 From: saimanikant Date: Fri, 29 Nov 2024 14:05:37 +0000 Subject: [PATCH 3/3] start dt_worker before download --- .../download_output_file.py | 42 ++++++++++--------- examples/python_large_output/project_setup.py | 7 +++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/examples/python_large_output/download_output_file.py b/examples/python_large_output/download_output_file.py index 24953afd4..877d03736 100644 --- a/examples/python_large_output/download_output_file.py +++ b/examples/python_large_output/download_output_file.py @@ -21,15 +21,12 @@ # SOFTWARE. """ -Example to query resources from a project. - -- Query values from evaluated jobs, computing some simple statistics on parameter values. -- Download files from the project - +Example to Download files of evaluated jobs from a project """ import argparse import logging import os +import tempfile import time from ansys.hps.client import Client, HPSError @@ -38,15 +35,23 @@ log = logging.getLogger(__name__) -def download_files(client, project_name): +def download_files(client, project_name, dir_path): """Download files.""" - out_path = os.path.join(os.path.dirname(__file__), "downloads") + + out_path = dir_path + if not out_path: + temp_dir = tempfile.TemporaryDirectory() + out_path = os.path.join(temp_dir.name, "downloads") + log.info(f"Downloading files to {out_path}") jms_api = JmsApi(client) - project = jms_api.get_project_by_name(name=project_name) + project = jms_api.get_project_by_name(name=project_name) project = jms_api.get_project(id=project.id) + # start dt client to time dowload of output files correctly + jms_api.client._start_dt_worker() + log.info(f"Project id: {project.id}") project_api = ProjectApi(client, project.id) @@ -54,10 +59,8 @@ def download_files(client, project_name): log.info(f"# evaluated jobs: {len(jobs)}") num = len(jobs) - log.info( - f"=== Example 1: Downloading output files of {num} jobs using ProjectApi.download_file()" - ) - for job in jobs[0:num]: + log.info(f"=== Downloading output files of {num} jobs using ProjectApi.download_file()") + for job in jobs: log.info(f"Job {job.id}") for task in project_api.get_tasks(job_id=job.id): log.info(f"Task {task.id}") @@ -65,10 +68,10 @@ def download_files(client, project_name): for f in files: fpath = os.path.join(out_path, f"task_{task.id}") log.info(f"Download output file {f.evaluation_path} to {fpath}") - start = time.process_time() - project_api.download_file(file=f, target_path=fpath) - log.info(f"Time taken to download output file: {(time.time() - start):.2f} seconds" - ) + start = time.time() + project_api.download_file(file=f, target_path=fpath) + log.info(f"Time taken to download output file: {(time.time() - start):.2f} seconds") + if __name__ == "__main__": @@ -77,17 +80,18 @@ def download_files(client, project_name): parser.add_argument("-U", "--url", default="https://127.0.0.1:8443/hps") parser.add_argument("-u", "--username", default="repuser") parser.add_argument("-p", "--password", default="repuser") + parser.add_argument("-dir", "--download-path", type=str) args = parser.parse_args() logger = logging.getLogger() - logging.basicConfig(format="%(message)s", level=logging.DEBUG) + logging.basicConfig(format="%(message)s", level=logging.INFO) try: log.info("Connect to HPC Platform Services") client = Client(url=args.url, username=args.username, password=args.password) log.info(f"HPS URL: {client.url}") - download_files(client=client, project_name=args.name) + download_files(client=client, project_name=args.name, dir_path=args.download_path) except HPSError as e: - log.error(str(e)) \ No newline at end of file + log.error(str(e)) diff --git a/examples/python_large_output/project_setup.py b/examples/python_large_output/project_setup.py index c79ec8446..a39baa6fb 100644 --- a/examples/python_large_output/project_setup.py +++ b/examples/python_large_output/project_setup.py @@ -188,6 +188,11 @@ def create_project(client, name, use_exec_script, python_version=None) -> Projec client = Client(url=args.url, username=args.username, password=args.password) try: - create_project(client, name=args.name, use_exec_script=args.use_exec_script, python_version=args.python_version) + create_project( + client, + name=args.name, + use_exec_script=args.use_exec_script, + python_version=args.python_version, + ) except HPSError as e: log.error(str(e))