From 68f9d038a49c8cc575b643c273f495f72b71f311 Mon Sep 17 00:00:00 2001 From: camila-camacho-phd Date: Mon, 15 Sep 2025 14:36:45 +0100 Subject: [PATCH 1/5] Initial structure of data handlers --- mmai25_hackathon/dataset.py | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/mmai25_hackathon/dataset.py b/mmai25_hackathon/dataset.py index 2121f48..c0db9da 100644 --- a/mmai25_hackathon/dataset.py +++ b/mmai25_hackathon/dataset.py @@ -14,6 +14,7 @@ from torch.utils.data import Dataset, Sampler from torch_geometric.data import DataLoader +import pandas as pd __all__ = ["BaseDataset", "BaseDataLoader", "BaseSampler"] @@ -72,6 +73,9 @@ def __add__(self, other): Note: This is not mandatory; treat it as a sketch you can refine or replace. """ raise NotImplementedError("Subclasses may implement __add__ method if needed.") + + def __subject_list__(self): + """Return a list with all the subject ids found in the dataset. """ @classmethod def prepare_data(cls, *args, **kwargs): @@ -109,6 +113,11 @@ class ECGDataset(BaseDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) +class ClinicalNotes(BaseDataset): + """Example subclass for a Clinical Notes dataset.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) class BaseDataLoader(DataLoader): """ @@ -132,15 +141,32 @@ class BaseDataLoader(DataLoader): Note: This is not a hard requirement. Consider it a future-facing idea you can evolve. """ - class MultimodalDataLoader(BaseDataLoader): """Example dataloader for handling multiple data modalities.""" def __init__(self, data_list, *args, **kwargs): super().__init__(*args, **kwargs) self.data_list = data_list + +class ID_Mapper(BaseDataLoader): + """ID Mapper for validating all IDs found accross the modalities list""" + def __init__(self, data_list, *args, **kwargs): + super().__init__(*args, **kwargs) + self.data_list = data_list + self.mapper = pd.DataFrame() + + def __create_mapper_(self): + """Function to creatre the mapper""" + self.mapper = pd.DataFrame() + + def __len__(self) -> int: + """Return the number of samples in the dataset.""" + raise NotImplementedError("Subclasses must implement __len__ method.") - + def __getitem__(self, idx: int): + """Return a single sample from the dataset.""" + raise NotImplementedError("Subclasses must implement __getitem__ method.") + class BaseSampler(Sampler): """ Base sampler to extend for custom sampling strategies. @@ -153,3 +179,15 @@ class BaseSampler(Sampler): balanced or paired sampling before passing to `BaseDataLoader`. Note: This is optional and meant as a design hint, not a constraint. """ + +class MultimodalDataSampler(BaseSampler): + """Example dataloader for handling multiple data modalities.""" + + def __init__(self, data_list, *args, **kwargs): + super().__init__(*args, **kwargs) + self.data_list = data_list + + def subject_extract(): + """Return all the info from a specific subject""" + return + From c74167f1ff91526ff448b84234a9b476d68860d1 Mon Sep 17 00:00:00 2001 From: camila-camacho-phd Date: Mon, 15 Sep 2025 14:58:46 +0100 Subject: [PATCH 2/5] RaiseErrors added --- mmai25_hackathon/dataset.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mmai25_hackathon/dataset.py b/mmai25_hackathon/dataset.py index c0db9da..7b8e0eb 100644 --- a/mmai25_hackathon/dataset.py +++ b/mmai25_hackathon/dataset.py @@ -52,6 +52,7 @@ def __getitem__(self, idx: int): def __repr__(self) -> str: """Return a string representation of the dataset.""" + return f"{self.__class__.__name__}({self.extra_repr()})" def extra_repr(self) -> str: @@ -105,19 +106,21 @@ class CXRDataset(BaseDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - + raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") class ECGDataset(BaseDataset): """Example subclass for an ECG dataset.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") class ClinicalNotes(BaseDataset): """Example subclass for a Clinical Notes dataset.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") class BaseDataLoader(DataLoader): """ @@ -157,7 +160,7 @@ def __init__(self, data_list, *args, **kwargs): def __create_mapper_(self): """Function to creatre the mapper""" - self.mapper = pd.DataFrame() + raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") def __len__(self) -> int: """Return the number of samples in the dataset.""" @@ -189,5 +192,6 @@ def __init__(self, data_list, *args, **kwargs): def subject_extract(): """Return all the info from a specific subject""" - return + raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") + From d8d5ed9be143df9b030774aca565f49be1c087ee Mon Sep 17 00:00:00 2001 From: camila-camacho-phd Date: Mon, 15 Sep 2025 15:29:17 +0100 Subject: [PATCH 3/5] __init__ implementation for load data library --- mmai25_hackathon/dataset.py | 4 ++++ mmai25_hackathon/load_data/__init__.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/mmai25_hackathon/dataset.py b/mmai25_hackathon/dataset.py index 7b8e0eb..cdf239a 100644 --- a/mmai25_hackathon/dataset.py +++ b/mmai25_hackathon/dataset.py @@ -15,6 +15,7 @@ from torch.utils.data import Dataset, Sampler from torch_geometric.data import DataLoader import pandas as pd +import load_data __all__ = ["BaseDataset", "BaseDataLoader", "BaseSampler"] @@ -107,6 +108,9 @@ class CXRDataset(BaseDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") + + def extract_cxr(): + load_data.load_chest_xray_image() class ECGDataset(BaseDataset): """Example subclass for an ECG dataset.""" diff --git a/mmai25_hackathon/load_data/__init__.py b/mmai25_hackathon/load_data/__init__.py index e69de29..3085035 100644 --- a/mmai25_hackathon/load_data/__init__.py +++ b/mmai25_hackathon/load_data/__init__.py @@ -0,0 +1,10 @@ +from . import cxr, text, ecg + +from .cxr import * +from .text import * +from .ecg import * + +__all__ = [] +__all__ += cxr.__all__ +__all__ += text.__all__ +__all__ += ecg.__all__ \ No newline at end of file From c4dd0548a0a1df72835f5e434109ca903ac79d27 Mon Sep 17 00:00:00 2001 From: camila-camacho-phd Date: Mon, 15 Sep 2025 15:46:06 +0100 Subject: [PATCH 4/5] pre-commit changes --- mmai25_hackathon/dataset.py | 34 +++++++++++++++----------- mmai25_hackathon/load_data/__init__.py | 11 ++++----- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/mmai25_hackathon/dataset.py b/mmai25_hackathon/dataset.py index cdf239a..76ff700 100644 --- a/mmai25_hackathon/dataset.py +++ b/mmai25_hackathon/dataset.py @@ -12,10 +12,10 @@ BaseSampler: Template for custom samplers, e.g., for multimodal sampling. """ +import load_data +import pandas as pd from torch.utils.data import Dataset, Sampler from torch_geometric.data import DataLoader -import pandas as pd -import load_data __all__ = ["BaseDataset", "BaseDataLoader", "BaseSampler"] @@ -53,7 +53,7 @@ def __getitem__(self, idx: int): def __repr__(self) -> str: """Return a string representation of the dataset.""" - + return f"{self.__class__.__name__}({self.extra_repr()})" def extra_repr(self) -> str: @@ -75,9 +75,9 @@ def __add__(self, other): Note: This is not mandatory; treat it as a sketch you can refine or replace. """ raise NotImplementedError("Subclasses may implement __add__ method if needed.") - + def __subject_list__(self): - """Return a list with all the subject ids found in the dataset. """ + """Return a list with all the subject ids found in the dataset.""" @classmethod def prepare_data(cls, *args, **kwargs): @@ -108,10 +108,11 @@ class CXRDataset(BaseDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") - - def extract_cxr(): + + def extract_cxr(sefl): load_data.load_chest_xray_image() + class ECGDataset(BaseDataset): """Example subclass for an ECG dataset.""" @@ -119,6 +120,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") + class ClinicalNotes(BaseDataset): """Example subclass for a Clinical Notes dataset.""" @@ -126,6 +128,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") + class BaseDataLoader(DataLoader): """ DataLoader for graph and non-graph data. @@ -148,24 +151,27 @@ class BaseDataLoader(DataLoader): Note: This is not a hard requirement. Consider it a future-facing idea you can evolve. """ + class MultimodalDataLoader(BaseDataLoader): """Example dataloader for handling multiple data modalities.""" def __init__(self, data_list, *args, **kwargs): super().__init__(*args, **kwargs) self.data_list = data_list - + + class ID_Mapper(BaseDataLoader): """ID Mapper for validating all IDs found accross the modalities list""" + def __init__(self, data_list, *args, **kwargs): super().__init__(*args, **kwargs) self.data_list = data_list self.mapper = pd.DataFrame() - + def __create_mapper_(self): """Function to creatre the mapper""" raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") - + def __len__(self) -> int: """Return the number of samples in the dataset.""" raise NotImplementedError("Subclasses must implement __len__ method.") @@ -173,7 +179,8 @@ def __len__(self) -> int: def __getitem__(self, idx: int): """Return a single sample from the dataset.""" raise NotImplementedError("Subclasses must implement __getitem__ method.") - + + class BaseSampler(Sampler): """ Base sampler to extend for custom sampling strategies. @@ -187,6 +194,7 @@ class BaseSampler(Sampler): Note: This is optional and meant as a design hint, not a constraint. """ + class MultimodalDataSampler(BaseSampler): """Example dataloader for handling multiple data modalities.""" @@ -194,8 +202,6 @@ def __init__(self, data_list, *args, **kwargs): super().__init__(*args, **kwargs) self.data_list = data_list - def subject_extract(): + def subject_extract(self): """Return all the info from a specific subject""" raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") - - diff --git a/mmai25_hackathon/load_data/__init__.py b/mmai25_hackathon/load_data/__init__.py index 3085035..c7370c3 100644 --- a/mmai25_hackathon/load_data/__init__.py +++ b/mmai25_hackathon/load_data/__init__.py @@ -1,10 +1,9 @@ -from . import cxr, text, ecg - -from .cxr import * -from .text import * -from .ecg import * +from . import cxr, ecg, text # noqa: F403 +from .cxr import * # noqa: F403 F401 +from .ecg import * # noqa: F403 F401 +from .text import * # noqa: F403 F401 __all__ = [] __all__ += cxr.__all__ __all__ += text.__all__ -__all__ += ecg.__all__ \ No newline at end of file +__all__ += ecg.__all__ From c20d2489ed9840369ebd08f8164f04cd2f70a4e0 Mon Sep 17 00:00:00 2001 From: camila-camacho-phd Date: Mon, 15 Sep 2025 17:45:53 +0100 Subject: [PATCH 5/5] Final Commit --- mmai25_hackathon/dataset.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mmai25_hackathon/dataset.py b/mmai25_hackathon/dataset.py index 76ff700..d105e80 100644 --- a/mmai25_hackathon/dataset.py +++ b/mmai25_hackathon/dataset.py @@ -17,7 +17,7 @@ from torch.utils.data import Dataset, Sampler from torch_geometric.data import DataLoader -__all__ = ["BaseDataset", "BaseDataLoader", "BaseSampler"] +__all__ = ["BaseDataset", "BaseDataLoader", "BaseSampler", "MultimodalDataLoader"] class BaseDataset(Dataset): @@ -107,6 +107,7 @@ class CXRDataset(BaseDataset): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + relative_path = "mimic-iv/mimic-cxr-jpg-chest-radiographs-with-structured-labels-2.1.0/" # noqa: F841 raise NotImplementedError("Subclasses may implement prepare_data class method if needed.") def extract_cxr(sefl): @@ -151,6 +152,16 @@ class BaseDataLoader(DataLoader): Note: This is not a hard requirement. Consider it a future-facing idea you can evolve. """ + def __init__(self, dataset, batch_size=1, shuffle=False, follow_batch=None, exclude_keys=None, **kwargs): + super().__init__( + dataset, + batch_size=batch_size, + shuffle=shuffle, + follow_batch=follow_batch, + exclude_keys=exclude_keys, + **kwargs, + ) + class MultimodalDataLoader(BaseDataLoader): """Example dataloader for handling multiple data modalities."""