Mlcube edit command

README.md

@@ -17,7 +17,7 @@ Inside this repo you can find all important pieces for running MedPerf. In its c
 
 If you use MedPerf, please cite our main paper: Karargyris, A., Umeton, R., Sheller, M.J. et al. Federated benchmarking of medical artificial intelligence with MedPerf. *Nature Machine Intelligence* **5**, 799–810 (2023). [https://www.nature.com/articles/s42256-023-00652-2](https://www.nature.com/articles/s42256-023-00652-2)
 
-Additonally, here you can see how others used MedPerf already: [https://scholar.google.com/scholar?q="medperf"](https://scholar.google.com/scholar?q="medperf").
+Additionally, here you can see how others used MedPerf already: [https://scholar.google.com/scholar?q="medperf"](https://scholar.google.com/scholar?q="medperf").
 
 ## Experiments
 

cli/cli_tests.sh

@@ -5,7 +5,6 @@
 ################### Start Testing ########################
 ##########################################################
 
-
 ##########################################################
 echo "=========================================="
 echo "Printing MedPerf version"
@@ -195,7 +194,7 @@ echo "Running data submission step"
 echo "====================================="
 print_eval "medperf dataset submit -p $PREP_UID -d $DIRECTORY/dataset_a -l $DIRECTORY/dataset_a --name='dataset_a' --description='mock dataset a' --location='mock location a' -y"
 checkFailed "Data submission step failed"
-DSET_A_UID=$(medperf dataset ls | grep dataset_a | tr -s ' ' | cut -d ' ' -f 1)
+DSET_A_UID=$(medperf dataset ls | grep dataset_a | tr -s ' ' | awk '{$1=$1;print}' | cut -d ' ' -f 1)
 echo "DSET_A_UID=$DSET_A_UID"
 ##########################################################
 

cli/medperf/cli.py

@@ -71,7 +71,7 @@ def execute(
             please run the command again with the --no-cache option.\n"""
         )
     else:
-        ResultSubmission.run(result.generated_uid, approved=approval)
+        ResultSubmission.run(result.local_id, approved=approval)
     config.ui.print("✅ Done!")
 
 

cli/medperf/commands/benchmark/benchmark.py

@@ -16,14 +16,16 @@
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local benchmarks"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered benchmarks"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user benchmarks"),
 ):
-    """List benchmarks stored locally and remotely from the user"""
+    """List benchmarks"""
     EntityList.run(
         Benchmark,
         fields=["UID", "Name", "Description", "State", "Approval Status", "Registered"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
     )
 
@@ -162,10 +164,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
+    unregistered: bool = typer.Option(
         False,
-        "--local",
-        help="Display local benchmarks if benchmark ID is not provided",
+        "--unregistered",
+        help="Display unregistered benchmarks if benchmark ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -180,4 +182,4 @@ def view(
     ),
 ):
     """Displays the information of one or more benchmarks"""
-    EntityView.run(entity_id, Benchmark, format, local, mine, output)
+    EntityView.run(entity_id, Benchmark, format, unregistered, mine, output)

cli/medperf/commands/benchmark/submit.py

@@ -79,7 +79,7 @@ def run_compatibility_test(self):
         self.ui.print("Running compatibility test")
         self.bmk.write()
         data_uid, results = CompatibilityTestExecution.run(
-            benchmark=self.bmk.generated_uid,
+            benchmark=self.bmk.local_id,
             no_cache=self.no_cache,
             skip_data_preparation_step=self.skip_data_preparation_step,
         )

cli/medperf/commands/compatibility_test/compatibility_test.py

@@ -95,7 +95,11 @@ def run(
 @clean_except
 def list():
     """List previously executed tests reports."""
-    EntityList.run(TestReport, fields=["UID", "Data Source", "Model", "Evaluator"])
+    EntityList.run(
+        TestReport,
+        fields=["UID", "Data Source", "Model", "Evaluator"],
+        unregistered=True,
+    )
 
 
 @app.command("view")
@@ -116,4 +120,4 @@ def view(
     ),
 ):
     """Displays the information of one or more test reports"""
-    EntityView.run(entity_id, TestReport, format, output=output)
+    EntityView.run(entity_id, TestReport, format, unregistered=True, output=output)

cli/medperf/commands/compatibility_test/run.py

@@ -239,7 +239,7 @@ def cached_results(self):
         """
         if self.no_cache:
             return
-        uid = self.report.generated_uid
+        uid = self.report.local_id
         try:
             report = TestReport.get(uid)
         except InvalidArgumentError:

cli/medperf/commands/compatibility_test/utils.py

@@ -138,23 +138,23 @@ def create_test_dataset(
     # TODO: existing dataset could make problems
     # make some changes since this is a test dataset
     config.tmp_paths.remove(data_creation.dataset.path)
-    data_creation.dataset.write()
     if skip_data_preparation_step:
         data_creation.make_dataset_prepared()
     dataset = data_creation.dataset
+    old_generated_uid = dataset.generated_uid
+    old_path = dataset.path
 
     # prepare/check dataset
     DataPreparation.run(dataset.generated_uid)
 
     # update dataset generated_uid
-    old_path = dataset.path
-    generated_uid = get_folders_hash([dataset.data_path, dataset.labels_path])
-    dataset.generated_uid = generated_uid
-    dataset.write()
-    if dataset.input_data_hash != dataset.generated_uid:
+    new_generated_uid = get_folders_hash([dataset.data_path, dataset.labels_path])
+    if new_generated_uid != old_generated_uid:
         # move to a correct location if it underwent preparation
-        new_path = old_path.replace(dataset.input_data_hash, generated_uid)
+        new_path = old_path.replace(old_generated_uid, new_generated_uid)
         remove_path(new_path)
         os.rename(old_path, new_path)
+        dataset.generated_uid = new_generated_uid
+        dataset.write()
 
-    return generated_uid
+    return new_generated_uid

cli/medperf/commands/dataset/dataset.py

@@ -17,17 +17,19 @@
 @app.command("ls")
 @clean_except
 def list(
-    local: bool = typer.Option(False, "--local", help="Get local datasets"),
+    unregistered: bool = typer.Option(
+        False, "--unregistered", help="Get unregistered datasets"
+    ),
     mine: bool = typer.Option(False, "--mine", help="Get current-user datasets"),
     mlcube: int = typer.Option(
         None, "--mlcube", "-m", help="Get datasets for a given data prep mlcube"
     ),
 ):
-    """List datasets stored locally and remotely from the user"""
+    """List datasets"""
     EntityList.run(
         Dataset,
         fields=["UID", "Name", "Data Preparation Cube UID", "State", "Status", "Owner"],
-        local_only=local,
+        unregistered=unregistered,
         mine_only=mine,
         mlcube=mlcube,
     )
@@ -149,8 +151,10 @@ def view(
         "--format",
         help="Format to display contents. Available formats: [yaml, json]",
     ),
-    local: bool = typer.Option(
-        False, "--local", help="Display local datasets if dataset ID is not provided"
+    unregistered: bool = typer.Option(
+        False,
+        "--unregistered",
+        help="Display unregistered datasets if dataset ID is not provided",
     ),
     mine: bool = typer.Option(
         False,
@@ -165,4 +169,4 @@ def view(
     ),
 ):
     """Displays the information of one or more datasets"""
-    EntityView.run(entity_id, Dataset, format, local, mine, output)
+    EntityView.run(entity_id, Dataset, format, unregistered, mine, output)

cli/medperf/commands/execution.py

@@ -47,23 +47,23 @@ def prepare(self):
         logging.debug(f"tmp results output: {self.results_path}")
 
     def __setup_logs_path(self):
-        model_uid = self.model.generated_uid
-        eval_uid = self.evaluator.generated_uid
-        data_hash = self.dataset.generated_uid
+        model_uid = self.model.local_id
+        eval_uid = self.evaluator.local_id
+        data_uid = self.dataset.local_id
 
         logs_path = os.path.join(
-            config.experiments_logs_folder, str(model_uid), str(data_hash)
+            config.experiments_logs_folder, str(model_uid), str(data_uid)
         )
         os.makedirs(logs_path, exist_ok=True)
         model_logs_path = os.path.join(logs_path, "model.log")
         metrics_logs_path = os.path.join(logs_path, f"metrics_{eval_uid}.log")
         return model_logs_path, metrics_logs_path
 
     def __setup_predictions_path(self):
-        model_uid = self.model.generated_uid
-        data_hash = self.dataset.generated_uid
+        model_uid = self.model.local_id
+        data_uid = self.dataset.local_id
         preds_path = os.path.join(
-            config.predictions_folder, str(model_uid), str(data_hash)
+            config.predictions_folder, str(model_uid), str(data_uid)
         )
         if os.path.exists(preds_path):
             msg = f"Found existing predictions for model {self.model.id} on dataset "

cli/medperf/commands/list.py

@@ -1,3 +1,5 @@
+from typing import List, Type
+from medperf.entities.interface import Entity
 from medperf.exceptions import InvalidArgumentError
 from tabulate import tabulate
 
@@ -8,29 +10,38 @@
 class EntityList:
     @staticmethod
     def run(
-        entity_class,
-        fields,
-        local_only: bool = False,
+        entity_class: Type[Entity],
+        fields: List[str],
+        unregistered: bool = False,
         mine_only: bool = False,
         **kwargs,
     ):
         """Lists all local datasets
 
         Args:
-            local_only (bool, optional): Display all local results. Defaults to False.
-            mine_only (bool, optional): Display all current-user results. Defaults to False.
+            unregistered (bool, optional): Display only local unregistered results. Defaults to False.
+            mine_only (bool, optional): Display all registered current-user results. Defaults to False.
             kwargs (dict): Additional parameters for filtering entity lists.
         """
-        entity_list = EntityList(entity_class, fields, local_only, mine_only, **kwargs)
+        entity_list = EntityList(
+            entity_class, fields, unregistered, mine_only, **kwargs
+        )
         entity_list.prepare()
         entity_list.validate()
         entity_list.filter()
         entity_list.display()
 
-    def __init__(self, entity_class, fields, local_only, mine_only, **kwargs):
+    def __init__(
+        self,
+        entity_class: Type[Entity],
+        fields: List[str],
+        unregistered: bool,
+        mine_only: bool,
+        **kwargs,
+    ):
         self.entity_class = entity_class
         self.fields = fields
-        self.local_only = local_only
+        self.unregistered = unregistered
         self.mine_only = mine_only
         self.filters = kwargs
         self.data = []
@@ -40,7 +51,7 @@ def prepare(self):
             self.filters["owner"] = get_medperf_user_data()["id"]
 
         entities = self.entity_class.all(
-            local_only=self.local_only, filters=self.filters
+            unregistered=self.unregistered, filters=self.filters
         )
         self.data = [entity.display_dict() for entity in entities]
 

cli/medperf/commands/mlcube/edit.py

@@ -0,0 +1,112 @@
+import logging
+from typing import Union
+
+import medperf.config as config
+from medperf.entities.cube import Cube
+from medperf.entities.edit_cube import EditCubeData
+
+
+class EditCube:
+    @classmethod
+    def run(cls, cube_uid: Union[str, int], mlcube_partial_info: EditCubeData):
+        """Update mlcube in the development mode on the medperf server
+
+        Args:
+            cube_uid: uid of cube to modify
+            mlcube_partial_info (dict): Dictionary containing the modified fields.
+        """
+        ui = config.ui
+
+        logging.debug("Downloading initial MLCube..")
+        edition = cls(cube_uid, mlcube_partial_info)
+        logging.debug("Validating MLCube DEVELOPMENT state..")
+        edition.validate_dev_state()
+
+        with ui.interactive():
+            ui.text = "Validating updated MLCube can be downloaded"
+            logging.debug("Applying MLCube edit..")
+            edition.apply_and_get_hashes()
+            ui.text = "Submitting MLCube edit to MedPerf"
+            logging.debug("Uploading MLCube..")
+            edition.upload()
+            edition.write()
+
+    def __init__(self, cube_uid: Union[str, int], edit_info: EditCubeData):
+        self.ui = config.ui
+        self.cube = Cube.get(cube_uid)
+        self.edit_info = edit_info
+
+    def validate_dev_state(self):
+        if self.cube.state != "DEVELOPMENT":
+            raise ValueError("Only cubes in development state can be edited")
+
+    def apply_and_get_hashes(self):
+        cube = self.cube
+        new = self.edit_info
+
+        if new.name:
+            cube.name = new.name
+
+        if new.git_mlcube_url:
+            cube.git_mlcube_url = new.git_mlcube_url
+            # Differs from further ifs: if mlcube.yaml url is provided, reset image also
+            cube.image_hash = ""
+
+        if new.mlcube_hash:
+            cube.mlcube_hash = new.mlcube_hash
+        elif new.git_mlcube_url is not None:
+            cube.mlcube_hash = ""
+
+        if new.git_parameters_url:
+            cube.git_parameters_url = new.git_parameters_url
+
+        if new.parameters_hash:
+            cube.parameters_hash = new.parameters_hash
+        elif new.git_parameters_url is not None:
+            cube.parameters_hash = ""
+
+        if new.image_tarball_url:
+            cube.image_tarball_url = new.image_tarball_url
+            # same as with git_mlcube_url
+            cube.image_hash = ""
+
+        if new.image_tarball_hash:
+            cube.image_tarball_hash = new.image_tarball_hash
+        elif new.image_tarball_url is not None:
+            cube.image_tarball_hash = ""
+
+        if new.additional_files_tarball_url:
+            cube.additional_files_tarball_url = new.additional_files_tarball_url
+
+        if new.additional_files_tarball_hash:
+            cube.additional_files_tarball_hash = new.additional_files_tarball_hash
+        elif new.additional_files_tarball_url is not None:
+            cube.additional_files_tarball_hash = ""
+
+        self.download()
+
+        if new.git_mlcube_url and not new.mlcube_hash:
+            new.mlcube_hash = cube.mlcube_hash
+        if new.git_parameters_url and not new.parameters_hash:
+            new.parameters_hash = cube.parameters_hash
+        if new.image_tarball_url and not new.image_tarball_hash:
+            new.image_tarball_hash = cube.image_tarball_hash
+        if new.additional_files_tarball_url and not new.additional_files_tarball_hash:
+            new.additional_files_tarball_hash = cube.additional_files_tarball_hash
+        if new.git_mlcube_url or new.image_tarball_url:
+            new.image_hash = cube.image_hash
+
+    def download(self):
+        logging.debug("removing from filesystem...")
+        self.cube.remove_from_filesystem()
+        logging.debug("download config files..")
+        self.cube.download_config_files()
+        logging.debug("download run files..")
+        self.cube.download_run_files()
+
+    def upload(self):
+        updated_body = Cube.edit(self.cube.id, self.edit_info)
+        self.cube = Cube(**updated_body)
+
+    def write(self):
+        self.cube.write()