From 451cd772dae810c1ac1de503e942eb3a8710318e Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 14:32:13 -0300 Subject: [PATCH 01/18] Add tutorials to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 212d8e13e..7ab518c2c 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ cython_debug/ # Dev Environment Specific .vscode .venv + +# Medperf Tutorials +medperf_tutorials/ \ No newline at end of file From aca297e88bb0df6f46e41e02d2ab931752d62c07 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 14:34:44 -0300 Subject: [PATCH 02/18] Standardise notation to Data Owner --- README.md | 2 +- docs/roles.md | 4 ++-- docs/what_is_medperf.md | 4 ++-- docs/workflow.md | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 550d281ca..a5b24cb27 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Additionally, here you can see how others used MedPerf already: [https://scholar ## Pilot Studies ## -MedPerf was also further utilized to support academic medical research on both public and private data through efforts across Dana-Farber Cancer Institute, Harvard T.H. Chan School of Public Health, University of Pennsylvania, Penn Medicine, University of Pennsylvania Health System, University of Strasbourg, Institute of Image-Guided Surgery (IHU Strasbourg), Fondazione Policlinico Universitario Agostino Gemelli IRCCS, University of California San Francisco, and other academic institutions. The figure below displays the data provider locations used in all pilot experiments. 🟢: Pilot 1 - Brain Tumor Segmentation Pilot Experiment; 🔴: Pilot 2 - Pancreas Segmentation Pilot Experiment. 🔵: Pilot 3 - Surgical Workflow Phase Recognition Pilot Experiment. Pilot 4 - Cloud Experiments, used data and processes from Pilot 1 and 2. +MedPerf was also further utilized to support academic medical research on both public and private data through efforts across Dana-Farber Cancer Institute, Harvard T.H. Chan School of Public Health, University of Pennsylvania, Penn Medicine, University of Pennsylvania Health System, University of Strasbourg, Institute of Image-Guided Surgery (IHU Strasbourg), Fondazione Policlinico Universitario Agostino Gemelli IRCCS, University of California San Francisco, and other academic institutions. The figure below displays the data owner locations used in all pilot experiments. 🟢: Pilot 1 - Brain Tumor Segmentation Pilot Experiment; 🔴: Pilot 2 - Pancreas Segmentation Pilot Experiment. 🔵: Pilot 3 - Surgical Workflow Phase Recognition Pilot Experiment. Pilot 4 - Cloud Experiments, used data and processes from Pilot 1 and 2. ![image](https://user-images.githubusercontent.com/25375373/163238058-6cf16f00-5238-4c80-8b58-d86f291a5bcf.png) diff --git a/docs/roles.md b/docs/roles.md index a09b536f6..5cca7bcb8 100644 --- a/docs/roles.md +++ b/docs/roles.md @@ -4,11 +4,11 @@ Here we introduce user roles at MedPerf. Depending on the objectives and expecta ## Benchmark Committee -May include healthcare stakeholders (e.g., hospitals, clinicians, patient advocacy groups, payors, etc.), regulatory bodies, data providers and model owners wishing to drive the evaluation of AI models on real world data. While the *Benchmark Committee* does not have admin privileges on MedPerf, they have elevated permissions regarding benchmark assets (e.g., task, evaluation metrics, etc.) and policies (e.g., participation of model owners, data providers, anonymizations) +May include healthcare stakeholders (e.g., hospitals, clinicians, patient advocacy groups, payors, etc.), regulatory bodies, data owners and model owners wishing to drive the evaluation of AI models on real world data. While the *Benchmark Committee* does not have admin privileges on MedPerf, they have elevated permissions regarding benchmark assets (e.g., task, evaluation metrics, etc.) and policies (e.g., participation of model owners, data owners, anonymizations) ![](./images/benchmark_committee.png) -## Data Providers +## Data Owners May include hospitals, medical practices, research organizations, and healthcare payors that own medical data, register medical data, and execute benchmarks. diff --git a/docs/what_is_medperf.md b/docs/what_is_medperf.md index c867a7ccf..aab65b313 100644 --- a/docs/what_is_medperf.md +++ b/docs/what_is_medperf.md @@ -4,7 +4,7 @@ td, th { border: none!important; } -MedPerf is an open-source framework for benchmarking medical ML models. It uses *Federated Evaluation* a method in which medical ML models are securely distributed to multiple global facilities for evaluation prioritizing patient privacy to mitigate legal and regulatory risks. The goal of *Federated Evaluation* is to make it simple and reliable to share ML models with many data providers, evaluate those ML models against their data in controlled settings, then aggregate and analyze the findings. +MedPerf is an open-source framework for benchmarking medical ML models. It uses *Federated Evaluation* a method in which medical ML models are securely distributed to multiple global facilities for evaluation prioritizing patient privacy to mitigate legal and regulatory risks. The goal of *Federated Evaluation* is to make it simple and reliable to share ML models with many data owners, evaluate those ML models against their data in controlled settings, then aggregate and analyze the findings. The MedPerf approach empowers healthcare stakeholders through neutral governance to assess and verify the performance of ML models in an efficient and human-supervised process without sharing any patient data across facilities during the process. @@ -19,7 +19,7 @@ The MedPerf approach empowers healthcare stakeholders through neutral governance MedPerf aims to identify bias and generalizability issues of medical ML models by evaluating them on diverse medical data across the world. This process allows developers of medical ML to efficiently identify performance and reliability issues on their models while healthcare stakeholders (e.g., hospitals, practices, etc.) can validate such models against clinical efficacy. -Importantly, MedPerf supports technology for **neutral governance** in order to enable **full trust** and **transparency** among participating parties (e.g., AI vendor, data provider, regulatory body, etc.). This is all encapsulated in the benchmark committee which is the overseeing body on a benchmark. +Importantly, MedPerf supports technology for **neutral governance** in order to enable **full trust** and **transparency** among participating parties (e.g., AI vendor, data owner, regulatory body, etc.). This is all encapsulated in the benchmark committee which is the overseeing body on a benchmark. | ![benchmark_committee.gif](images/benchmark_committee.gif) | |:--:| diff --git a/docs/workflow.md b/docs/workflow.md index 55b87a291..9e84aca5e 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -4,7 +4,7 @@ +Currently, the MedPerf administration is the only one able to create users, controlling access to the system and permissions to own a benchmark. For example, if a hospital (Data Owner), a model owner, or a benchmark committee wants to have access to MedPerf, they need to contact the MedPerf administrator to add a user. --> -A benchmark in MedPerf is a collection of assets that are developed by the benchmark committee that aims to evaluate medical ML on decentralized data providers. +A benchmark in MedPerf is a collection of assets that are developed by the benchmark committee that aims to evaluate medical ML on decentralized data owners. The process is simple yet effective enabling scalability. @@ -24,7 +24,7 @@ The benchmarking process starts with establishing a benchmark committee of healt +The benchmark committee recruits Data Owners and Model Owners either by inviting trusted parties or by making an open call for participation. A higher number of dataset providers recruited can maximize diversity on a global scale. --> ## Step 2. Register Benchmark @@ -32,14 +32,14 @@ The benchmark committee recruits Data Providers and Model Owners either by invit ## Step 3. Register Dataset -Data Providers that want to be part of the benchmark can [register their own datasets, prepare them, and associate them](getting_started/data_owner_demo.md) with the benchmark. A dataset will be prepared using the benchmark's Data Preparator MLCube and the dataset's **metadata** is registered within the MedPerf server. +Data Owners that want to be part of the benchmark can [register their own datasets, prepare them, and associate them](getting_started/data_owner_demo.md) with the benchmark. A dataset will be prepared using the benchmark's Data Preparator MLCube and the dataset's **metadata** is registered within the MedPerf server. | ![flow_preparation.gif](images/flow_preparation_association_folders.PNG) | |:--:| | *Data Preparation* | -The data provider then can request to participate in the benchmark with their dataset. Requesting the association will run the benchmark's reference workflow to assure the compatibility of the prepared dataset structure with the workflow. Once the association request is approved by the Benchmark Committee, then the dataset becomes a part of the benchmark. +The data owner then can request to participate in the benchmark with their dataset. Requesting the association will run the benchmark's reference workflow to assure the compatibility of the prepared dataset structure with the workflow. Once the association request is approved by the Benchmark Committee, then the dataset becomes a part of the benchmark. ![](./images/dataset_preparation_association.png) @@ -51,9 +51,9 @@ Once a benchmark is submitted by the Benchmark Committee, any user can [submit t ## Step 5. Execute Benchmark -The Benchmark Committee may notify Data Providers that models are available for benchmarking. Data Providers can then [run the benchmark models](getting_started/data_owner_demo.md#5-execute-the-benchmark) locally on their data. +The Benchmark Committee may notify Data Owners that models are available for benchmarking. Data Owners can then [run the benchmark models](getting_started/data_owner_demo.md#5-execute-the-benchmark) locally on their data. -This procedure retrieves the model MLCubes associated with the benchmark and runs them on the indicated prepared dataset to generate predictions. The Metrics MLCube of the benchmark is then retrieved to evaluate the predictions. Once the evaluation results are generated, the data provider can [submit them](getting_started/data_owner_demo.md#6-submit-a-result) to the platform. +This procedure retrieves the model MLCubes associated with the benchmark and runs them on the indicated prepared dataset to generate predictions. The Metrics MLCube of the benchmark is then retrieved to evaluate the predictions. Once the evaluation results are generated, the data owner can [submit them](getting_started/data_owner_demo.md#6-submit-a-result) to the platform. ![](./images/execution_flow_folders.PNG) From 7e1b5613fc0262a24977a35aa41395b08205511f Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 14:40:17 -0300 Subject: [PATCH 03/18] Explicitly mention what is the workspace directory in Section 2, similar to what is done in section 4 --- docs/getting_started/benchmark_owner_demo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started/benchmark_owner_demo.md b/docs/getting_started/benchmark_owner_demo.md index bd654f54a..c5e8a4e1f 100644 --- a/docs/getting_started/benchmark_owner_demo.md +++ b/docs/getting_started/benchmark_owner_demo.md @@ -54,7 +54,7 @@ A demo dataset is a small reference dataset. It contains a few data records and 2. When a model owner wants to participate in the benchmark, the MedPerf client tests the compatibility of their model with the benchmark's data preparation cube and metrics cube. The test is run using the benchmark's demo dataset as input. -For this tutorial, you are provided with a demo dataset for the chest X-ray classification workflow. The dataset can be found in your workspace folder under `demo_data`. It is a small dataset comprising two chest X-ray images and corresponding thoracic disease labels. +For this tutorial, you are provided with a demo dataset for the chest X-ray classification workflow. The dataset can be found in your workspace folder (`medperf_tutorial`) under `demo_data`. It is a small dataset comprising two chest X-ray images and corresponding thoracic disease labels. You can test the workflow now that you have the three MLCubes and the demo data. Testing the workflow before submitting any asset to the MedPerf server is usually recommended. From 2e51ebb318ff2624ab55a1e5ca63344113b4954d Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 14:42:21 -0300 Subject: [PATCH 04/18] Mention venv in local server installation --- docs/getting_started/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started/setup.md b/docs/getting_started/setup.md index 3b4ae0c9d..170efe038 100644 --- a/docs/getting_started/setup.md +++ b/docs/getting_started/setup.md @@ -10,7 +10,7 @@ If this is your first time using MedPerf, install the MedPerf client library as For this tutorial, you should spawn a local MedPerf server for the MedPerf client to communicate with. Note that this server will be hosted on your `localhost` and not on the internet. -1. Install the server requirements ensuring you are in MedPerf's root folder: +1. Install the server requirements ensuring you are in MedPerf's root folder. If a virtual environment was created when installing the MedPerf client (see [this section](installation.md#install-medperf)), make sure the same virtual environment is used when installing the local server dependencies: ```bash pip install -r server/requirements.txt From 925a57f3f370b0abd60dfaea1ebcd379c9fb289b Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 15:25:01 -0300 Subject: [PATCH 05/18] Fix medperf_tutorial in .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7ab518c2c..8c6c979b7 100644 --- a/.gitignore +++ b/.gitignore @@ -149,4 +149,4 @@ cython_debug/ .venv # Medperf Tutorials -medperf_tutorials/ \ No newline at end of file +medperf_tutorial/ \ No newline at end of file From f5e41397c99842ecd9606944d0deefa9bc836ce0 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 15:55:08 -0300 Subject: [PATCH 06/18] Conditional logout if already logged in for tutorial scripts --- tutorials_scripts/medperf_login.sh | 13 +++++++++++++ tutorials_scripts/setup_benchmark_tutorial.sh | 4 +++- tutorials_scripts/setup_data_tutorial.sh | 3 ++- tutorials_scripts/setup_model_tutorial.sh | 3 ++- .../simulate_data_association_approval.sh | 4 ++-- 5 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 tutorials_scripts/medperf_login.sh diff --git a/tutorials_scripts/medperf_login.sh b/tutorials_scripts/medperf_login.sh new file mode 100644 index 000000000..1d575489d --- /dev/null +++ b/tutorials_scripts/medperf_login.sh @@ -0,0 +1,13 @@ +LOGIN_EMAIL=$1 +AUTH_STATUS=$(medperf auth status) + +ALREADY_LOGGED_EMAIL=$(echo $AUTH_STATUS | grep -Eho "[[:graph:]]+@[[:graph:]]+") + +if [ ! -z $ALREADY_LOGGED_EMAIL ] +then + echo "Logging out of current logged in e-mail $ALREADY_LOGGED_EMAIL" + medperf auth logout +fi + +echo "Logging into email $LOGIN_EMAIL for this tutorial" +medperf auth login -e $LOGIN_EMAIL \ No newline at end of file diff --git a/tutorials_scripts/setup_benchmark_tutorial.sh b/tutorials_scripts/setup_benchmark_tutorial.sh index 5d0b94415..0cb4f6125 100644 --- a/tutorials_scripts/setup_benchmark_tutorial.sh +++ b/tutorials_scripts/setup_benchmark_tutorial.sh @@ -1,4 +1,5 @@ # Create a workspace +original_dir=$(echo $PWD) mkdir -p medperf_tutorial cd medperf_tutorial @@ -26,4 +27,5 @@ sh download.sh rm download.sh ## Login locally as benchmark owner -medperf auth login -e testbo@example.com \ No newline at end of file +cd $original_dir +sh tutorials_scripts/medperf_login.sh testbo@example.com \ No newline at end of file diff --git a/tutorials_scripts/setup_data_tutorial.sh b/tutorials_scripts/setup_data_tutorial.sh index ddbc412b9..57a3fd973 100644 --- a/tutorials_scripts/setup_data_tutorial.sh +++ b/tutorials_scripts/setup_data_tutorial.sh @@ -15,4 +15,5 @@ tar -xf $filename rm $filename ## Login locally as data owner -medperf auth login -e testdo@example.com \ No newline at end of file +cd .. +sh tutorials_scripts/medperf_login.sh testdo@example.com \ No newline at end of file diff --git a/tutorials_scripts/setup_model_tutorial.sh b/tutorials_scripts/setup_model_tutorial.sh index fbcb97485..d0a95ed7d 100644 --- a/tutorials_scripts/setup_model_tutorial.sh +++ b/tutorials_scripts/setup_model_tutorial.sh @@ -11,4 +11,5 @@ sh download.sh rm download.sh ## Login locally as model owner -medperf auth login -e testmo@example.com \ No newline at end of file +cd .. +sh tutorials_scripts/medperf_login.sh testdo@example.com \ No newline at end of file diff --git a/tutorials_scripts/simulate_data_association_approval.sh b/tutorials_scripts/simulate_data_association_approval.sh index 3a77b6a23..f0f8881b4 100644 --- a/tutorials_scripts/simulate_data_association_approval.sh +++ b/tutorials_scripts/simulate_data_association_approval.sh @@ -1,3 +1,3 @@ -medperf auth login -e testbo@example.com +sh tutorials_scripts/medperf_login.sh testdo@example.com medperf association approve -b 1 -d 1 -medperf auth login -e testdo@example.com +sh tutorials_scripts/medperf_login.sh testdo@example.com From fb2fffbf1a89763fd3c83bcb4ea4a4611aeeff08 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Tue, 22 Apr 2025 16:33:10 -0300 Subject: [PATCH 07/18] Single cleanup script. Also fixes pointing to wrong directory in cleanup --- docs/getting_started/shared/cleanup.md | 18 ++---------------- tutorials_scripts/tutorials_cleanup.sh | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 tutorials_scripts/tutorials_cleanup.sh diff --git a/docs/getting_started/shared/cleanup.md b/docs/getting_started/shared/cleanup.md index 5b0fcba5a..188c6d324 100644 --- a/docs/getting_started/shared/cleanup.md +++ b/docs/getting_started/shared/cleanup.md @@ -4,21 +4,7 @@ You have reached the end of the tutorial! If you are planning to rerun any of th - To shut down the local MedPerf server: press `CTRL`+`C` in the terminal where the server is running. -- To cleanup the downloaded files workspace (make sure you are in the MedPerf's root directory): +- To cleanup the downloaded files workspace, MedPerf server database and test storage, run the following script (make sure you are in the MedPerf's root directory): ```bash -rm -fr medperf_tutorial -``` - -- To cleanup the local MedPerf server database: (make sure you are in the MedPerf's root directory) - -```bash -cd server -sh reset_db.sh -``` - -- To cleanup the test storage: - -```bash -rm -fr ~/.medperf/localhost_8000 -``` +sh tutorials_scripts/tutorials_cleanup.sh diff --git a/tutorials_scripts/tutorials_cleanup.sh b/tutorials_scripts/tutorials_cleanup.sh new file mode 100644 index 000000000..f8b95c717 --- /dev/null +++ b/tutorials_scripts/tutorials_cleanup.sh @@ -0,0 +1,25 @@ +# This script should be run from the medperf too directory, ie the parent directory to where this file is +# sh tutorials_scripts/tutorials_cleanup.sh + +# Remove medperf_tutorial directory created by tutorials +echo "Removing medperf_tutorial directory from tutorials..." +rm -rf medperf_tutorial + +# Cleanup local server database +echo "Reseting local server database..." +cd server +sh reset_db.sh + +# Clean up test storage + +echo "Removing local storage from tutorials..." +for dir in ~/.medperf/* +do + if [ -d "$dir" ] + then + rm -rf "$dir"/localhost_8000 + fi +done + +# Also delete demo directory +rm -rf ~/.medperf/demo \ No newline at end of file From e33a5f7873c6b62ff5a68b8bd29c9fd44d629ad3 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 08:55:03 -0300 Subject: [PATCH 08/18] Explicitly mention cleanup refers to the local MedPerf server and also explicitly mention local test storage --- docs/getting_started/shared/cleanup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started/shared/cleanup.md b/docs/getting_started/shared/cleanup.md index 188c6d324..35bc4f48a 100644 --- a/docs/getting_started/shared/cleanup.md +++ b/docs/getting_started/shared/cleanup.md @@ -4,7 +4,7 @@ You have reached the end of the tutorial! If you are planning to rerun any of th - To shut down the local MedPerf server: press `CTRL`+`C` in the terminal where the server is running. -- To cleanup the downloaded files workspace, MedPerf server database and test storage, run the following script (make sure you are in the MedPerf's root directory): +- To cleanup the downloaded files workspace, local MedPerf server database and local test storage, run the following script (make sure you are in the MedPerf's root directory): ```bash sh tutorials_scripts/tutorials_cleanup.sh From e286a4c1563a5ba272f252eaf355e26c5bfa80b8 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 10:01:12 -0300 Subject: [PATCH 09/18] Close code block --- docs/getting_started/shared/cleanup.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started/shared/cleanup.md b/docs/getting_started/shared/cleanup.md index 35bc4f48a..def118809 100644 --- a/docs/getting_started/shared/cleanup.md +++ b/docs/getting_started/shared/cleanup.md @@ -8,3 +8,4 @@ You have reached the end of the tutorial! If you are planning to rerun any of th ```bash sh tutorials_scripts/tutorials_cleanup.sh +``` \ No newline at end of file From fa3cf2dd041a1ab9643e26f00241212a53ce270a Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 10:03:45 -0300 Subject: [PATCH 10/18] Fix final cd to properly call medperf_login script --- tutorials_scripts/setup_model_tutorial.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tutorials_scripts/setup_model_tutorial.sh b/tutorials_scripts/setup_model_tutorial.sh index d0a95ed7d..df77dec7d 100644 --- a/tutorials_scripts/setup_model_tutorial.sh +++ b/tutorials_scripts/setup_model_tutorial.sh @@ -1,4 +1,5 @@ # Create a workspace +original_dir=$(echo $PWD) mkdir -p medperf_tutorial cd medperf_tutorial @@ -11,5 +12,5 @@ sh download.sh rm download.sh ## Login locally as model owner -cd .. +cd $original_dir sh tutorials_scripts/medperf_login.sh testdo@example.com \ No newline at end of file From e7f6a0f9063bb7f44bdeba9e46a75781f305e06a Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 10:12:03 -0300 Subject: [PATCH 11/18] Change task name so it matches what is in the MLCube --- .../chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml b/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml index 49262f5f1..c11e63212 100644 --- a/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml +++ b/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml @@ -15,7 +15,7 @@ docker: build_file: "Dockerfile" tasks: - infer: + evaluate: # Computes predictions on input data parameters: inputs: From 87d60555700f7b86d2ced0c34c03ae457c64b333 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 10:28:45 -0300 Subject: [PATCH 12/18] Revert "Change task name so it matches what is in the MLCube" This reverts commit e7f6a0f9063bb7f44bdeba9e46a75781f305e06a. --- .../chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml b/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml index c11e63212..49262f5f1 100644 --- a/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml +++ b/examples/chestxray_tutorial/model_mobilenetv2/mlcube/mlcube.yaml @@ -15,7 +15,7 @@ docker: build_file: "Dockerfile" tasks: - evaluate: + infer: # Computes predictions on input data parameters: inputs: From c6d77d15ba1b9093c083a986584444da0b1b11e2 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 11:09:24 -0300 Subject: [PATCH 13/18] Update model UID --- docs/getting_started/model_owner_demo.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/model_owner_demo.md b/docs/getting_started/model_owner_demo.md index 42b46375b..706bb234d 100644 --- a/docs/getting_started/model_owner_demo.md +++ b/docs/getting_started/model_owner_demo.md @@ -117,12 +117,12 @@ Benchmark workflows are run by Data Owners, who will get notified when a new mod To initiate an association request, you need to collect the following information: - The target benchmark ID, which is `1` -- The server UID of your MLCube, which is `4`. +- The server UID of your MLCube, which is `5`. Run the following command to request associating your MLCube with the benchmark: ```bash -medperf mlcube associate --benchmark 1 --model_uid 4 +medperf mlcube associate --benchmark 1 --model_uid 5 ``` This command will first run the benchmark's workflow on your model to ensure your model is compatible with the benchmark workflow. Then, the association request information is printed on the screen, which includes an executive summary of the test mentioned. You will be prompted to confirm sending this information and initiating this association request. From 957fee3488c339c9a2e53b5d86ea018116e10f22 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 11:22:03 -0300 Subject: [PATCH 14/18] Fix user for association approval simulation --- tutorials_scripts/simulate_data_association_approval.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials_scripts/simulate_data_association_approval.sh b/tutorials_scripts/simulate_data_association_approval.sh index f0f8881b4..8b5e4374e 100644 --- a/tutorials_scripts/simulate_data_association_approval.sh +++ b/tutorials_scripts/simulate_data_association_approval.sh @@ -1,3 +1,3 @@ -sh tutorials_scripts/medperf_login.sh testdo@example.com +sh tutorials_scripts/medperf_login.sh testbo@example.com medperf association approve -b 1 -d 1 sh tutorials_scripts/medperf_login.sh testdo@example.com From 3cb878b0768a5412a95c91f35ef63490c77f3922 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 11:36:41 -0300 Subject: [PATCH 15/18] Update MLCube IDs --- docs/getting_started/benchmark_owner_demo.md | 12 ++++++------ docs/getting_started/data_owner_demo.md | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/getting_started/benchmark_owner_demo.md b/docs/getting_started/benchmark_owner_demo.md index c5e8a4e1f..a24c82206 100644 --- a/docs/getting_started/benchmark_owner_demo.md +++ b/docs/getting_started/benchmark_owner_demo.md @@ -277,9 +277,9 @@ You need to keep at hand the following information: ``` - For this tutorial, the UIDs are as follows: - - Data preparator UID: `1` - - Reference model UID: `2` - - Evaluator UID: `3` + - Data preparator UID: `2` + - Reference model UID: `3` + - Evaluator UID: `4` You can create and submit your benchmark using the following command: @@ -288,9 +288,9 @@ medperf benchmark submit \ --name tutorial_bmk \ --description "MedPerf demo bmk" \ --demo-url "{{ demo_url }}" \ - --data-preparation-mlcube 1 \ - --reference-model-mlcube 2 \ - --evaluator-mlcube 3 \ + --data-preparation-mlcube 2 \ + --reference-model-mlcube 3 \ + --evaluator-mlcube 4 \ --operational ``` diff --git a/docs/getting_started/data_owner_demo.md b/docs/getting_started/data_owner_demo.md index f5c5558e9..00b2246d0 100644 --- a/docs/getting_started/data_owner_demo.md +++ b/docs/getting_started/data_owner_demo.md @@ -151,8 +151,8 @@ After running the command, you will receive a summary of the executions. You wil ```text model local result UID partial result from cache error ------- ------------------ ---------------- ------------ ------- - 2 b1m2d1 False True - 4 b1m4d1 False False + 5 b1m5d1 False False + 3 b1m3d1 False True Total number of models: 2 1 were skipped (already executed), of which 0 have partial results 0 failed @@ -164,12 +164,12 @@ Total number of models: 2 This means that the benchmark has two models: - A model that you already ran when you requested the association. This explains why it was skipped. -- Another model that ran successfully. Its result generated UID is `b1m4d1`. +- Another model that ran successfully. Its result generated UID is `b1m5d1`. You can view the results by running the following command with the specific local result UID. For example: ```bash -medperf result view b1m4d1 +medperf result view b1m5d1 ``` For now, your results are only local. Next, you will learn how to submit the results. @@ -179,10 +179,10 @@ For now, your results are only local. Next, you will learn how to submit the res ![Dataset Owner submits evaluation results](../tutorial_images/do-6-do-submits-eval-results.png){class="tutorial-sticky-image-content"} After executing the benchmark, you will submit a result to the MedPerf server. To do so, you have to find the target result generated UID. -As an example, you will be submitting the result of UID `b1m4d1`. To do this, run the following command: +As an example, you will be submitting the result of UID `b1m5d1`. To do this, run the following command: ```bash -medperf result submit --result b1m4d1 +medperf result submit --result b1m5d1 ``` The information that is going to be submitted will be printed to the screen and you will be prompted to confirm that you want to submit. From 0676462dae19e771884adf5d89ad588e71caebe2 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 11:45:56 -0300 Subject: [PATCH 16/18] Better formattingat the end of section 4 --- docs/getting_started/data_owner_demo.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/getting_started/data_owner_demo.md b/docs/getting_started/data_owner_demo.md index 00b2246d0..aa6b16448 100644 --- a/docs/getting_started/data_owner_demo.md +++ b/docs/getting_started/data_owner_demo.md @@ -130,7 +130,11 @@ _For the sake of continuing the tutorial only_, run the following to simulate th sh tutorials_scripts/simulate_data_association_approval.sh ``` -You can verify if your association request has been approved by running `medperf association ls -bd`. +You can verify if your association request has been approved by running the following command: + +```bash +medperf association ls -bd +``` ## 5. Execute the Benchmark From b81e5818539e950756d665e557aee14cbe243754 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 11:48:26 -0300 Subject: [PATCH 17/18] Update MLCube hashes to match DockerHub --- server/seed_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/seed_utils.py b/server/seed_utils.py index 915138649..df491d313 100644 --- a/server/seed_utils.py +++ b/server/seed_utils.py @@ -110,7 +110,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "parameters_hash": "1541e05437040745d2489e8d2cf14795d4839eecc15c1ac959c84f6b77c1a5df", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "4cefa8a2b9580220a0503076f1e961e4d86ec72dad8e1e78b9c43444dee9a4cd", + "image_hash": "b40bc0be41c9f6ade37c10fd794455993699025d4fcaa02691aa6a378350a7da", "additional_files_tarball_url": "", "additional_files_tarball_hash": "", "metadata": {}, @@ -154,7 +154,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "chestxray_tutorial/cnn_weights.tar.gz" ), "additional_files_tarball_hash": "bff003e244759c3d7c8b9784af0819c7f252da8626745671ccf7f46b8f19a0ca", - "image_hash": "63d48be95598a7474d1bd26fdee41dcc20a752a436c28b35dbd33796087a6d29", + "image_hash": "4e06daf08803589f1c0c0ae42dbfce1c986d99a5e556db7dc32e0f4ea1fa194b", "image_tarball_url": "", "image_tarball_hash": "", "metadata": {}, @@ -195,7 +195,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "parameters_hash": "16cad451c54b801a5b50d999330465d7f68ab5f6d30a0674268d2d17c7f26b73", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "2dbea6a3ba40d553905427c8bb156f219970306f55061462918fd19b220e9b51", + "image_hash": "2a4ef53c9edfb41ca0e7f44319023d553e2a9778a94feaf5c9b74872e30e4da4", "additional_files_tarball_url": "", "additional_files_tarball_hash": "", "metadata": {}, @@ -287,7 +287,7 @@ def create_model(api_server, model_owner_token, benchmark_owner_token, benchmark "additional_files_tarball_hash": "771f67bba92a11c83d16a522f0ba1018020ff758e2277d33f49056680c788892", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "7ae4a8ecbe899b5486c699d934d74bce7f3aa73d779e0138e6d119cd8040b46e", + "image_hash": "efafb940a1b11c9b18f57f69cb8320ef777fdb5da2eb19d17348730661100a45", "metadata": {}, }, "id", From b8310dd51ab3bb69ddbca2d54f14043167fb7771 Mon Sep 17 00:00:00 2001 From: Bruno Rodrigues Date: Wed, 23 Apr 2025 16:18:54 -0300 Subject: [PATCH 18/18] Revert "Update MLCube hashes to match DockerHub" This reverts commit b81e5818539e950756d665e557aee14cbe243754. --- server/seed_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/seed_utils.py b/server/seed_utils.py index df491d313..915138649 100644 --- a/server/seed_utils.py +++ b/server/seed_utils.py @@ -110,7 +110,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "parameters_hash": "1541e05437040745d2489e8d2cf14795d4839eecc15c1ac959c84f6b77c1a5df", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "b40bc0be41c9f6ade37c10fd794455993699025d4fcaa02691aa6a378350a7da", + "image_hash": "4cefa8a2b9580220a0503076f1e961e4d86ec72dad8e1e78b9c43444dee9a4cd", "additional_files_tarball_url": "", "additional_files_tarball_hash": "", "metadata": {}, @@ -154,7 +154,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "chestxray_tutorial/cnn_weights.tar.gz" ), "additional_files_tarball_hash": "bff003e244759c3d7c8b9784af0819c7f252da8626745671ccf7f46b8f19a0ca", - "image_hash": "4e06daf08803589f1c0c0ae42dbfce1c986d99a5e556db7dc32e0f4ea1fa194b", + "image_hash": "63d48be95598a7474d1bd26fdee41dcc20a752a436c28b35dbd33796087a6d29", "image_tarball_url": "", "image_tarball_hash": "", "metadata": {}, @@ -195,7 +195,7 @@ def create_benchmark(api_server, benchmark_owner_token, admin_token): "parameters_hash": "16cad451c54b801a5b50d999330465d7f68ab5f6d30a0674268d2d17c7f26b73", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "2a4ef53c9edfb41ca0e7f44319023d553e2a9778a94feaf5c9b74872e30e4da4", + "image_hash": "2dbea6a3ba40d553905427c8bb156f219970306f55061462918fd19b220e9b51", "additional_files_tarball_url": "", "additional_files_tarball_hash": "", "metadata": {}, @@ -287,7 +287,7 @@ def create_model(api_server, model_owner_token, benchmark_owner_token, benchmark "additional_files_tarball_hash": "771f67bba92a11c83d16a522f0ba1018020ff758e2277d33f49056680c788892", "image_tarball_url": "", "image_tarball_hash": "", - "image_hash": "efafb940a1b11c9b18f57f69cb8320ef777fdb5da2eb19d17348730661100a45", + "image_hash": "7ae4a8ecbe899b5486c699d934d74bce7f3aa73d779e0138e6d119cd8040b46e", "metadata": {}, }, "id",