diff --git a/examples/overview.json b/examples/overview.json index 2082a796..2060d231 100644 --- a/examples/overview.json +++ b/examples/overview.json @@ -1,57 +1,80 @@ { "leaves":{ - "fibonacci": { + "default|gaya":{ "contents":[{ "type":"plot", "ref":"aggregated_data", "plot":{ - "title":"Performance over time", + "title":"Performance over time (Max over parameters)", "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ {"column":"perfvalue","agg":"filter:elapsed"} ], "xaxis":{ "parameter":"date", "label":"Date" }, "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"parameter":"n","label": "N" }, - "color_axis":{"parameter":"method","label":"Method"} + "aggregations":[{"column":"elements|n|mesh|discretization|resources.tasks|tasks","agg":"max"}], + "color_axis":{"parameter":"perfvalue","label":"Performance Variable"} } }] }, - "sortingApp":[ { - "title":"Performance over time", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"parameter":"elements","label": "N" }, - "color_axis":{"parameter":"algorithm","label":"Method"} - }], - "parallelSum":[{ - "title":"Performance over time", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:computation_time"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"parameter":"elements","label": "N" }, - "color_axis":{"parameter":"tasks","label":"Tasks"} - }], - "multiplication|matrixvector":[{ - "title":"Performance over time", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed_compute"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"parameter":"elements","label": "N" }, - "color_axis":{"parameter":"tasks","label":"Tasks"} - }] + + "Fibonacci":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance over time (Max over method)", + "plot_types":["scatter"], + "aggregations":[ {"column":"method","agg":"max"}], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"n","label":"N"}, + "secondary_axis":{"parameter":"perfvalue","label":"Performance Variable"} + } + }] + }, + "parallel_sum":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance over time (Max over elements)", + "plot_types":["scatter"], + "aggregations":[ {"column":"elements","agg":"max"} ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"tasks","label":"Tasks"}, + "secondary_axis":{"parameter":"perfvalue","label":"Performance Variable"} + } + }] + }, + "Multiplication":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance over time (Max over elements)", + "plot_types":["scatter"], + "aggregations":[ {"column":"elements","agg":"max"} ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"tasks","label":"Tasks"}, + "secondary_axis":{"parameter":"perfvalue","label":"Performance Variable"} + } + }] + }, + "sorting":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance over time (Max over elements)", + "plot_types":["scatter"], + "aggregations":[ {"column":"elements","agg":"max"} ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"algorithm","label":"Algorithm"}, + "secondary_axis":{"parameter":"perfvalue","label":"Performance Variable"} + } + }] + } }, "machines":{ "machines":{ @@ -61,7 +84,6 @@ "plot":{ "title":"Benchmarks by machine", "plot_types":["sunburst"], - "transformation":"performance", "aggregations":[ {"column":"leaves","agg":"nunique"} ], "xaxis":{"parameter":"applications","label":"Application"}, "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, @@ -70,133 +92,166 @@ } }] }, - "parallelSum":[{ - "title":"Maximum Performance over time", - "plot_types":["scatter","marked_scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:computation_time"}, - {"column":"elements","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"label":"Tasks","parameter":"tasks"}, - "color_axis":{"parameter":"machines","label":"Machine"} - }], - "fibonacci":[{ - "title":"Maximum Performance over time", - "plot_types":["scatter","marked_scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed"}, - {"column":"n","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"label":"Method","parameter":"method"}, - "color_axis":{"parameter":"machines","label":"Machine"} - }], - "sortingApp": [ { - "title":"Maximum Performance over time", - "plot_types":["scatter","marked_scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed"}, - {"column":"elements","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "secondary_axis":{"label":"Method","parameter":"algorithm"}, - "color_axis":{"parameter":"machines","label":"Machine"} - }], - "multiplication":[{ - "title":"Performance over time by machine", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed_compute"}, - {"column":"elements|tasks","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "color_axis":{"parameter":"machines","label": "Machine" } - }] + "Fibonacci":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'elapsed' over time by machine", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:elapsed"}, + {"column":"n","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"machines","label":"Machine"}, + "secondary_axis":{"parameter":"use_cases","label":"Use Case"} + } + }] + }, + "parallel_sum":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'computation_time' over time by machine", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:computation_time"}, + {"column":"tasks","agg":"max"}, + {"column":"elements","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"machines","label":"Machine"}, + "secondary_axis":{"parameter":"use_cases","label":"Use Case"} + } + }] + }, + "sorting|Multiplication":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'elapsed_compute' over time by machine", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:elapsed_compute"}, + {"column":"elements","agg":"max"}, + {"column":"algorithm|tasks","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"machines","label":"Machine"}, + "secondary_axis":{"parameter":"use_cases","label":"Use Case"} + } + }] + } }, "applications": { - "applications":[{ - "title":"Benchmarks by application", - "plot_types":["sunburst"], - "transformation":"performance", - "aggregations":[ - {"column":"leaves","agg":"nunique"} - ], - "xaxis":{"parameter":"machines","label":"Machine"}, - "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, - "color_axis":{"label":"Date","parameter":"date"}, - "secondary_axis":{"parameter":"applications","label":"Application"} - }], - "gaya|discoverer":[ - { - "title":"Status over time (pass/fail) ", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"first"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"result"}, - "color_axis":{"parameter":"applications","label":"Application"} - }, - { - "title":"Benchmarks per application", - "plot_types":["sunburst"], - "transformation":"performance", - "aggregations":[ - {"column":"leaves","agg":"nunique"} - ], - "xaxis":{ "parameter":"applications", "label":"Application" }, - "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, - "color_axis":{"parameter":"date","label":"Date"} - } - ], - "multiplication":[{ - "title":"Performance over time by application", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed_compute"}, - {"column":"elements|tasks","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "color_axis":{"parameter":"applications","label":"Application"} - }] + "applications":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Benchmarks by application", + "plot_types":["sunburst"], + "aggregations":[ {"column":"leaves","agg":"nunique"} ], + "xaxis":{"parameter":"machines","label":"Machine"}, + "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, + "color_axis":{"label":"Date","parameter":"date"}, + "secondary_axis":{"parameter":"applications","label":"Application"} + } + }] + }, + "gaya|discoverer|default":{ + "contents":[ + { + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Status over time (pass/fail) ", + "plot_types":["scatter"], + "aggregations":[ {"column":"perfvalue","agg":"first"} ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"result"}, + "color_axis":{"parameter":"applications","label":"Application"} + } + }, + { + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Benchmarks per application", + "plot_types":["sunburst"], + "aggregations":[ + {"column":"leaves","agg":"nunique"} + ], + "xaxis":{ "parameter":"applications", "label":"Application" }, + "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, + "color_axis":{"parameter":"date","label":"Date"} + } + } + ] + } }, "use_cases": { - "gaya":[ - { - "title":"Benchmarks per use case", - "plot_types":["sunburst"], - "transformation":"performance", - "aggregations":[ - {"column":"leaves","agg":"nunique"} - ], - "xaxis":{ "parameter":"use_cases", "label":"Use Case" }, - "yaxis": {"label":"Number of benchmarks", "parameter":"runs.num_cases"}, - "color_axis":{"parameter":"applications","label":"Application"} - } - ], - "matrixvector|use_cases":[{ - "title":"Performance over time by use case", - "plot_types":["scatter"], - "transformation":"performance", - "aggregations":[ - {"column":"perfvalue","agg":"filter:elapsed_compute"}, - {"column":"elements|tasks","agg":"max"} - ], - "xaxis":{ "parameter":"date", "label":"Date" }, - "yaxis": {"label":"Execution time (s)", "parameter":"value"}, - "color_axis":{"parameter":"use_cases","label":"Use Case"} - }] + "fibonacci":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'elapsed' over time by use case", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:elapsed"}, + {"column":"n","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"use_cases","label":"Use Case"}, + "secondary_axis":{"parameter":"machine","label":"Machine"} + } + }] + }, + "parallelSum":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'computation_time' over time by use case", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:computation_time"}, + {"column":"tasks","agg":"max"}, + {"column":"elements","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"use_cases","label":"Use Case"}, + "secondary_axis":{"parameter":"machine","label":"Machines"} + } + }] + }, + "sortingApp|matrixvector":{ + "contents":[{ + "type":"plot", + "ref":"aggregated_data", + "plot":{ + "title":"Performance of 'elapsed_compute' over time by use case", + "plot_types":["scatter"], + "aggregations":[ + {"column":"perfvalue","agg":"filter:elapsed_compute"}, + {"column":"elements","agg":"max"}, + {"column":"algorithm|tasks","agg":"max"} + ], + "xaxis":{ "parameter":"date", "label":"Date" }, + "yaxis": {"label":"Execution time (s)", "parameter":"value"}, + "color_axis":{"parameter":"use_cases","label":"Use Case"}, + "secondary_axis":{"parameter":"machines","label":"Machines"} + } + }] + } } } \ No newline at end of file diff --git a/examples/website_config.json b/examples/website_config.json index dcb5699e..862fa9a2 100644 --- a/examples/website_config.json +++ b/examples/website_config.json @@ -10,11 +10,12 @@ "template_defaults":{ "repositories":{ "template": "${TEMPLATE_DIR}/overview.adoc.j2", - "data":{"filepath":"./reports/overview.json","prefix":"plots"} + "data":{"filepath":"./examples/overview.json","prefix":"plots"} }, "components":{ "all": { - "template": "${TEMPLATE_DIR}/overview.adoc.j2" + "template": "${TEMPLATE_DIR}/overview.adoc.j2", + "data":{"filepath":"./examples/overview.json","prefix":"plots"} }, "machines": { "template": "${TEMPLATE_DIR}/machineOverview.adoc.j2", @@ -50,37 +51,36 @@ "parallelSum": {"title": "parallelSum", "description": "Compute the sum of an array using MPI","card_image":"fa-solid fa-plus" } }, "use_cases":{ - "multiplication": {"title":"multiplication", "description":"Multiplication between different objects", "card_image":"fa-solid fa-times"} + "Fibonacci": { "title": "Fibonacci", "description": "n-th Fibonacci number", "card_image": "fa-solid fa-fan" }, + "Multiplication": { "title": "Multiplication", "description": "Multiplication between different objects", "card_image": "fa-solid fa-times" }, + "parallel_sum": { "title": "Parallel_Sum", "description": "Compute the sum of an array using MPI", "card_image": "fa-solid fa-plus" }, + "sorting": { "title": "Sorting", "description": "Sorting algorithms", "card_image": "fa-solid fa-arrow-up-wide-short" } } - }, "component_map": { - "component_order": [ "machines", "applications", "use_cases" ], - "mapping": { - "gaya": { - "sortingApp": { - "path": "./reports/sortingApp/sorting/gaya", - "template_info": { "application": "sortingApp", "machine": "gaya" } - }, - "fibonacci": { - "path": "./reports/fibonacci/Fibonacci/gaya", - "template_info": { "application": "fibonacci", "machine": "gaya" } - }, - "matrixvector": { - "multiplication":{ - "path": "./reports/matrixvector/Multiplication/gaya", - "template_info": { "application": "matrixvector", "machine": "gaya", "use_case":"multiplication"} - } - }, - "parallelSum": { - "path": "./reports/parallelSum/parallel_sum/gaya", - "template_info": { "application": "parallelSum", "machine": "gaya" } + "default": { + "sortingApp": { + "sorting":{ + "path": "./reports/sortingApp/sorting/default", + "template_info": { "application": "sortingApp", "machine": "default" } } }, - "discoverer":{ - "fibonacci": { - "path": "./reports/fibonacci/Fibonacci/discoverer", - "template_info": { "application": "fibonacci", "machine": "discoverer" } + "fibonacci": { + "Fibonacci":{ + "path": "./reports/fibonacci/Fibonacci/default", + "template_info": { "application": "fibonacci", "machine": "default" } + } + }, + "matrixvector": { + "Multiplication":{ + "path": "./reports/matrixvector/Multiplication/default", + "template_info": { "application": "matrixvector", "machine": "default", "use_case":"multiplication"} + } + }, + "parallelSum": { + "parallel_sum":{ + "path": "./reports/parallelSum/parallel_sum/default", + "template_info": { "application": "parallelSum", "machine": "default" } } } } diff --git a/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py b/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py index aa0f1bc8..7612aa3b 100644 --- a/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py +++ b/src/feelpp/benchmarking/dashboardRenderer/schemas/dashboardSchema.py @@ -27,14 +27,27 @@ class TemplateDataFile(BaseModel): prefix: Optional[str] = "" format: str = None action: Optional[str] = "input" + platform: Optional[str] = "local" filepath: str @model_validator(mode="after") def checkFormat(self): - if self.format is None: - self.format = self.filepath.split(".")[-1] + if self.platform == "local": + if self.format is None: + self.format = self.filepath.split(".")[-1] + else: + if self.format is None: + raise ValueError("Format must be specified for remote data") return self + @field_validator("platform",mode="after") + @classmethod + def checkPlatform(cls,v): + if v not in ["local","girder"]: + raise NotImplementedError(f"Data management platform not supported : {v}") + return v + + class TemplateInfo(BaseModel): template:Optional[str] = None data: Union[ @@ -230,5 +243,5 @@ def setTemplateDefaults(self): if not component_data.template: component_data.template = template - + component_data.data = component_data.data[::-1] #Put defaults at the start so actual values overwrite return self \ No newline at end of file diff --git a/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py b/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py index 8eb21458..7bbb7c63 100644 --- a/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py +++ b/src/feelpp/benchmarking/dashboardRenderer/views/templateDataHandler.py @@ -1,7 +1,7 @@ - +from feelpp.benchmarking.dashboardRenderer.handlers.girder import GirderHandler from feelpp.benchmarking.dashboardRenderer.schemas.dashboardSchema import TemplateInfo,TemplateDataFile from feelpp.benchmarking.json_report.renderer import JsonReportController -import json, os, warnings +import json, os, warnings, tempfile from typing import Union, Dict,Any class DataHandler: @@ -49,7 +49,15 @@ def extractData( self, data: TemplateDataFile, partials:Dict[str,Any] = {}, extr UserWarning: If a file specified by `filepath` does not exist and the action is "input". JSONDecodeError: If the file content is not valid JSON. """ - filepath = os.path.join( self.template_data_dir, data.filepath ) if self.template_data_dir else data.filepath + if data.platform != "local": + tmpdir = tempfile.mkdtemp() + if data.platform == "girder": + download_handler = GirderHandler( tmpdir ) + download_handler.downloadFile(data.filepath,name=data.filepath) + local_filepath = os.path.join(tmpdir,data.filepath) + filepath = local_filepath + else : + filepath = os.path.join( self.template_data_dir, data.filepath ) if self.template_data_dir else data.filepath if data.action == "input": if not os.path.exists( filepath ): warnings.warn(f"{filepath} does not exist. Skipping") @@ -60,6 +68,9 @@ def extractData( self, data: TemplateDataFile, partials:Dict[str,Any] = {}, extr if not content: content = "{}" template_data = json.loads(content) + + if data.platform != "local": + os.remove(local_filepath) if data.prefix: return {data.prefix: template_data} return template_data diff --git a/src/feelpp/benchmarking/report/templates/overview.adoc.j2 b/src/feelpp/benchmarking/report/templates/overview.adoc.j2 index 8218a22c..93991041 100644 --- a/src/feelpp/benchmarking/report/templates/overview.adoc.j2 +++ b/src/feelpp/benchmarking/report/templates/overview.adoc.j2 @@ -12,7 +12,7 @@ {% for key,key_plots in plots_by_child_repo.items() %} {%if self_id in key.split("|") %} - {% with report_data = {"aggregated_data":aggregated.get("data")}, report = ValidateJsonReport(key_plots)%} + {% with report_data = {"aggregated_data":{"aggregated_data":aggregated.get("data")}}, report = ValidateJsonReport(key_plots)%} {% include "json2adoc_report.adoc.j2" %} {% endwith %}