From ccf78e550fffef37cc01ef6b661b4cb5cdaf9540 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Fri, 8 Jul 2022 09:41:06 -0400 Subject: [PATCH 1/4] List individual tools in the readme --- .github/scripts/update_readme.py | 26 ++++++++++++++++++++++---- .github/templates/README.md.j2 | 18 +++++++++++------- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/scripts/update_readme.py b/.github/scripts/update_readme.py index e7b85a73c8..3e67023f17 100644 --- a/.github/scripts/update_readme.py +++ b/.github/scripts/update_readme.py @@ -1,6 +1,7 @@ from datetime import datetime import json import sys +import yaml from jinja2 import Template @@ -28,13 +29,30 @@ template = Template(f.read()) with open(readme_path, "w") as f: - htmlout = "Chunk IDTool ListLatest reportPrevious report" ids = list(chunks.keys()) ids = [int(x) for x in ids] ids.sort() ids = [str(x) for x in ids] + # Write the chunked test reports table + reports_html = "Chunk IDTool ListLatest reportPrevious report" for eachid in ids: eachchunk = chunks.get(eachid) - htmlout += "{}Toolset{}{}".format(eachid, eachchunk.get("tools"), eachchunk.get("run1"), eachchunk.get("date1"), eachchunk.get("run2"), eachchunk.get("date2")) - htmlout += "" - f.write(template.render(anviltools=htmlout, reportdir=readme_path.replace("/README.md", ""))) + reports_html += "{}Toolset{}{}".format( + eachid, eachchunk.get("tools"), eachchunk.get("run1"), eachchunk.get("date1"), eachchunk.get("run2"), eachchunk.get("date2")) + reports_html += "" + # List tools individually + tool_tests_html = "#Tool IDTest results" + count = 0 + for each_id in ids: + chunk = chunks.get(each_id) + chunk_tools_url = chunk.get('tools') + chunk_tools_path = chunk_tools_url.replace('https://github.com/anvilproject/galaxy-tests/blob/main/', '') + with open(chunk_tools_path, 'r') as tf: + chunk_tools = yaml.safe_load(tf) + for tool in chunk_tools['tools']: + for rev in tool['revisions']: + count += 1 + tool_tests_html += f"{count}{tool['owner']}/{tool['name']}@{rev}" + tool_tests_html += "" + + f.write(template.render(toolreports=reports_html, tooltests=tool_tests_html, reportdir=readme_path.replace("/README.md", ""))) diff --git a/.github/templates/README.md.j2 b/.github/templates/README.md.j2 index cb173306ee..b0016cf22f 100644 --- a/.github/templates/README.md.j2 +++ b/.github/templates/README.md.j2 @@ -1,17 +1,21 @@ # Automated Tests for Galaxy on Kubernetes Stacks -## Galaxy on GKE deployed via GalaxyKubeMan (AnVIL) -### Deployment Testing -Twice a day, [GalaxyKubeMan (GKM)](https://github.com/galaxyproject/galaxykubeman-helm) is deployed on GKE, mimicking an AnVIL deployment. The purpose of these tests is to provide reasonable confidence that Galaxy is launchable on the AnVIL everyday. +## Deployment testing +Twice a day, [GalaxyKubeMan (GKM)](https://github.com/galaxyproject/galaxykubeman-helm) is deployed on GKE, mimicking an AnVIL deployment. The purpose of these tests is to provide reasonable confidence that Galaxy can be launched on the AnVIL everyday. Below is a plot summarizing successful deployments and GKM install times. Click here or on the image for more details. -### Tool Testing -After each successful deployment, automated tool tests are also run against the instance. These serve as an end-to-end-like test for Galaxy, providing confidence that Galaxy is not only launchable but functional. These tests cycle on a weekly basis through the entire suite of tools installed by default on AnVIL, providing reasonable confidence that the tools encountered by most users remain functional, and automating the detection and reporting of tools breaking. +## Tool testing +After each successful deployment, automated tool tests are also run against the instance. These serve as an end-to-end-like test for Galaxy, providing confidence that Galaxy deployments are functional. These tests cycle on a weekly basis through the entire suite of tools installed by default on AnVIL, providing reasonable confidence that the tools encountered by most users remain functional, and automating the detection and reporting of tools breaking. -Latest tool tests for each chunk: +### Tool test reports -{{ anviltools }}
+Latest tool test reports for each chunk: +{{ toolreports }}
+ +### Inidividual tool tests results over time: + +{{ tooltests }}
From 380e14792697080ad9920a2b9d0df67f2154edaa Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Fri, 8 Jul 2022 12:26:33 -0400 Subject: [PATCH 2/4] Add a template for including multiple tests per tool --- .github/scripts/update_readme.py | 31 +++++++++++++++++++++++++++---- .github/templates/README.md.j2 | 25 +++++++++++++++++++------ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/scripts/update_readme.py b/.github/scripts/update_readme.py index 3e67023f17..bf404532b1 100644 --- a/.github/scripts/update_readme.py +++ b/.github/scripts/update_readme.py @@ -37,11 +37,16 @@ reports_html = "Chunk IDTool ListLatest reportPrevious report" for eachid in ids: eachchunk = chunks.get(eachid) - reports_html += "{}Toolset{}{}".format( - eachid, eachchunk.get("tools"), eachchunk.get("run1"), eachchunk.get("date1"), eachchunk.get("run2"), eachchunk.get("date2")) + reports_html += f""" + + {eachid} + Toolset + {eachchunk.get("date1")} + {eachchunk.get("date2")} + """ reports_html += "" # List tools individually - tool_tests_html = "#Tool IDTest results" + tool_tests_html = "#Tool IDTest #Test results" count = 0 for each_id in ids: chunk = chunks.get(each_id) @@ -52,7 +57,25 @@ for tool in chunk_tools['tools']: for rev in tool['revisions']: count += 1 - tool_tests_html += f"{count}{tool['owner']}/{tool['name']}@{rev}" + tool_tests_html += f""" + + {count} + {tool['owner']}/{tool['name']}@{rev}""" + test_count = 0 + for test in range(3): # TODO: Link with tests + test_count += 1 + if test_count == 1: + tool_tests_html += f""" + Test #{test_count} + FP""" + else: + tool_tests_html += f""" + + + Test #{test_count} + FP + """ + tool_tests_html +="" tool_tests_html += "" f.write(template.render(toolreports=reports_html, tooltests=tool_tests_html, reportdir=readme_path.replace("/README.md", ""))) diff --git a/.github/templates/README.md.j2 b/.github/templates/README.md.j2 index b0016cf22f..8f89a15a34 100644 --- a/.github/templates/README.md.j2 +++ b/.github/templates/README.md.j2 @@ -1,14 +1,27 @@ # Automated Tests for Galaxy on Kubernetes Stacks + ## Deployment testing -Twice a day, [GalaxyKubeMan (GKM)](https://github.com/galaxyproject/galaxykubeman-helm) is deployed on GKE, mimicking an AnVIL deployment. The purpose of these tests is to provide reasonable confidence that Galaxy can be launched on the AnVIL everyday. -Below is a plot summarizing successful deployments and GKM install times. -Click here or on the image for more details. +Twice a day, [GalaxyKubeMan +(GKM)](https://github.com/galaxyproject/galaxykubeman-helm) is deployed on GKE, +mimicking an AnVIL deployment. The purpose of these tests is to provide +reasonable confidence that Galaxy can be launched on the AnVIL everyday. + +Below is a plot summarizing successful deployments and GKM install times. Click +here or on the image for more details. - + + + ## Tool testing -After each successful deployment, automated tool tests are also run against the instance. These serve as an end-to-end-like test for Galaxy, providing confidence that Galaxy deployments are functional. These tests cycle on a weekly basis through the entire suite of tools installed by default on AnVIL, providing reasonable confidence that the tools encountered by most users remain functional, and automating the detection and reporting of tools breaking. +After each successful deployment, automated tool tests are also run against the +instance. These serve as an end-to-end-like test for Galaxy, providing +confidence that Galaxy deployments are functional. These tests cycle on a weekly +basis through the entire suite of tools installed by default on AnVIL, providing +reasonable confidence that the tools encountered by most users remain +functional, and automating the detection and reporting of tools breaking. ### Tool test reports @@ -16,6 +29,6 @@ Latest tool test reports for each chunk: {{ toolreports }}
-### Inidividual tool tests results over time: +### Individual tool tests results over time: {{ tooltests }}
From cf51bd6d4ad8b0f5124594e38ea0a7bc523464b9 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Fri, 8 Jul 2022 12:45:29 -0400 Subject: [PATCH 3/4] Add tests chunk id and link to the latest report --- .github/scripts/update_readme.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/update_readme.py b/.github/scripts/update_readme.py index bf404532b1..720bf0ce11 100644 --- a/.github/scripts/update_readme.py +++ b/.github/scripts/update_readme.py @@ -46,7 +46,7 @@ """ reports_html += "" # List tools individually - tool_tests_html = "#Tool IDTest #Test results" + tool_tests_html = "#Chunk #Tool IDTest #Test results" count = 0 for each_id in ids: chunk = chunks.get(each_id) @@ -60,6 +60,7 @@ tool_tests_html += f""" {count} + {each_id} {tool['owner']}/{tool['name']}@{rev}""" test_count = 0 for test in range(3): # TODO: Link with tests @@ -71,7 +72,7 @@ else: tool_tests_html += f""" - + Test #{test_count} FP """ From 5e44c4fdbcb2af1eddd0fb1a642e319fbbe68e1a Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Fri, 8 Jul 2022 12:50:10 -0400 Subject: [PATCH 4/4] Call it a report, not chunk --- .github/scripts/update_readme.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/update_readme.py b/.github/scripts/update_readme.py index 720bf0ce11..9a355cf954 100644 --- a/.github/scripts/update_readme.py +++ b/.github/scripts/update_readme.py @@ -34,7 +34,7 @@ ids.sort() ids = [str(x) for x in ids] # Write the chunked test reports table - reports_html = "Chunk IDTool ListLatest reportPrevious report" + reports_html = "Report IDTool ListLatest reportPrevious report" for eachid in ids: eachchunk = chunks.get(eachid) reports_html += f""" @@ -46,7 +46,7 @@ """ reports_html += "" # List tools individually - tool_tests_html = "#Chunk #Tool IDTest #Test results" + tool_tests_html = "#Report IDTool IDTest #Test results" count = 0 for each_id in ids: chunk = chunks.get(each_id)