From 73a2acb85ed2f9fa01fefa228404afc0a06e0cac Mon Sep 17 00:00:00 2001 From: vitous Date: Sat, 31 Jan 2026 01:01:15 +0100 Subject: [PATCH 1/4] add contains and makedirs to dataset --- brukerapi/dataset.py | 12 ++++++++++-- brukerapi/jcampdx.py | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/brukerapi/dataset.py b/brukerapi/dataset.py index 842915c..cfeeb49 100644 --- a/brukerapi/dataset.py +++ b/brukerapi/dataset.py @@ -166,7 +166,7 @@ def __init__(self, path, **state): raise FileNotFoundError(self.path) # directory constructor - if self.path.is_dir(): + if self.path.is_dir() and state.get('load'): content = os.listdir(self.path) if 'fid' in content: self.path = self.path / 'fid' @@ -214,6 +214,14 @@ def __getitem__(self, item): raise KeyError(item) + + def __contains__(self, item): + for parameter_file in self._parameters.values(): + if item in parameter_file: + return True + return False + + def __call__(self, **kwargs): self._set_state(kwargs) return self @@ -593,7 +601,7 @@ def write(self, path, **kwargs): parent = path.parent if not parent.exists(): - os.mkdir(parent) + os.makedirs(parent,exist_ok=True) self._write_parameters(parent) self._write_data(path) diff --git a/brukerapi/jcampdx.py b/brukerapi/jcampdx.py index 18dd724..fbfc081 100644 --- a/brukerapi/jcampdx.py +++ b/brukerapi/jcampdx.py @@ -633,6 +633,12 @@ def __add__(self, other): def __getitem__(self, key): return self.params[key] + + def __contains__(self, item): + if item in self.params: + return True + return False + def __delitem__(self, key): del self.params[key] From 4290d2bd36c986308c5797f62d141b7dd2d6c267 Mon Sep 17 00:00:00 2001 From: vitous Date: Sat, 31 Jan 2026 01:02:01 +0100 Subject: [PATCH 2/4] fix splitters --- brukerapi/splitters.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/brukerapi/splitters.py b/brukerapi/splitters.py index 7b12026..97af706 100644 --- a/brukerapi/splitters.py +++ b/brukerapi/splitters.py @@ -1,9 +1,10 @@ from .utils import index_to_slice from .dataset import Dataset - +import os import numpy as np import copy from pathlib import Path +from .exceptions import MissingProperty SUPPORTED_FG = ['FG_ISA','FG_IRMODE','FG_ECHO'] @@ -40,7 +41,7 @@ def _split_VisuCoreDataMin(self, dataset, visu_pars, select, fg_rel_index): VisuCoreDataMin = visu_pars['VisuCoreDataMin'] value = np.reshape(VisuCoreDataMin.value, dataset.shape_final[dataset.encoded_dim:], order='F') value = value[index_to_slice(select, value.shape, fg_rel_index)] - VisuCoreDataMin.size = (np.prod(value.shape),) + VisuCoreDataMin.size = (int(np.prod(value.shape)),) VisuCoreDataMin.value = value.flatten(order='F') def _split_VisuCoreDataMax(self, dataset, visu_pars, select, fg_rel_index): @@ -54,7 +55,7 @@ def _split_VisuCoreDataMax(self, dataset, visu_pars, select, fg_rel_index): VisuCoreDataMax = visu_pars['VisuCoreDataMax'] value = np.reshape(VisuCoreDataMax.value, dataset.shape_final[dataset.encoded_dim:], order='F') value = value[index_to_slice(select, value.shape, fg_rel_index)] - VisuCoreDataMax.size = (np.prod(value.shape),) + VisuCoreDataMax.size = (int(np.prod(value.shape)),) VisuCoreDataMax.value = value.flatten(order='F') def _split_VisuCoreDataOffs(self, dataset, visu_pars, select, fg_rel_index): @@ -68,7 +69,7 @@ def _split_VisuCoreDataOffs(self, dataset, visu_pars, select, fg_rel_index): VisuCoreDataOffs = visu_pars['VisuCoreDataOffs'] value = np.reshape(VisuCoreDataOffs.value, dataset.shape_final[dataset.encoded_dim:],order='F') value = value[index_to_slice(select, value.shape, fg_rel_index)] - VisuCoreDataOffs.size = (np.prod(value.shape),) + VisuCoreDataOffs.size = (int(np.prod(value.shape)),) VisuCoreDataOffs.value = value.flatten(order='F') def _split_VisuCoreDataSlope(self, dataset, visu_pars, select, fg_rel_index): @@ -82,7 +83,7 @@ def _split_VisuCoreDataSlope(self, dataset, visu_pars, select, fg_rel_index): VisuCoreDataSlope = visu_pars['VisuCoreDataSlope'] value = np.reshape(VisuCoreDataSlope.value, dataset.shape_final[dataset.encoded_dim:],order='F') value = value[index_to_slice(select, value.shape, fg_rel_index)] - VisuCoreDataSlope.size = (np.prod(value.shape),) + VisuCoreDataSlope.size = (int(np.prod(value.shape)),) VisuCoreDataSlope.value = value.flatten(order='F') def _split_VisuCoreTransposition(self, dataset, visu_pars, index, fg_index): @@ -93,7 +94,7 @@ def _split_VisuCoreTransposition(self, dataset, visu_pars, index, fg_index): value = np.reshape(VisuCoreTransposition.value, dataset.shape_final[dataset.encoded_dim:], order='F') value = value[index_to_slice(index, value.shape, fg_index - dataset.encoded_dim)] - VisuCoreTransposition.size = (np.prod(value.shape),) + VisuCoreTransposition.size = (int(np.prod(value.shape)),) VisuCoreTransposition.value = value.flatten(order='F') @@ -154,9 +155,12 @@ def split(self, dataset, select=None, write=False, path_out=None, **kwargs): for select_ in select: # construct a new Dataset, without loading data, the data will be supplied later name = '{}_{}_{}/2dseq'.format(dataset.path.parents[0].name, self.fg, select_) - + + dset_path = dataset.path.parents[1] / name + os.makedirs(dset_path,exist_ok=True) + # construct a new Dataset, without loading data, the data will be supplied later - dataset_ = Dataset(dataset.path.parents[1] / name, load=False) + dataset_ = Dataset(dataset.path.parents[1] / name, load=0) dataset_.parameters = self._split_params(dataset, select_, fg_abs_index, fg_rel_index, fg_size) @@ -250,7 +254,7 @@ def _split_VisuFGElemComment(self, visu_pars, fg_scheme, index, fg_index): def _split_VisuAcqEchoTime(self, visu_pars, select): VisuAcqEchoTime = visu_pars['VisuAcqEchoTime'] - value = VisuAcqEchoTime.value + value = VisuAcqEchoTime.list VisuAcqEchoTime.size=(1,) VisuAcqEchoTime.value = float(value[select]) @@ -272,9 +276,10 @@ def split(self, dataset, write=False, path_out=None): """ try: - VisuCoreSlicePacksSlices = dataset['VisuCoreSlicePacksSlices'].value + VisuCoreSlicePacksSlices = dataset['VisuCoreSlicePacksSlices'].nested except KeyError: - print('Parameter VisuCoreSlicePacksSlices not found') + raise MissingProperty('Parameter VisuCoreSlicePacksSlices not found') + # list of split data sets datasets = [] @@ -299,9 +304,10 @@ def split(self, dataset, write=False, path_out=None): # name of the data set created by the split name = '{}_sp_{}/2dseq'.format(dataset.path.parents[0].name, sp_index) + os.makedirs(dataset.path.parents[1] / name,exist_ok=True) # construct a new Dataset, without loading data, the data will be supplied later - dataset_ = Dataset(dataset.path.parents[1] / name, load=False) + dataset_ = Dataset(dataset.path.parents[1] / name, load=0) # SPLIT parameters dataset_.parameters = self._split_parameters(dataset, frame_range, fg_rel_index, fg_abs_index, sp_index, frame_count) @@ -337,7 +343,8 @@ def _split_parameters(self, dataset, frame_range, fg_rel_index, fg_abs_index, sp self._split_VisuCoreDataMax(dataset, visu_pars_, frame_range, fg_rel_index) self._split_VisuCoreDataOffs(dataset, visu_pars_, frame_range, fg_rel_index) self._split_VisuCoreDataSlope(dataset, visu_pars_, frame_range, fg_rel_index) - self._split_VisuCoreTransposition(dataset, visu_pars_, frame_range, fg_rel_index) + if "VisuCoreTransposition" in dataset: + self._split_VisuCoreTransposition(dataset, visu_pars_, frame_range, fg_rel_index) self._split_VisuCoreFrameCount(dataset, visu_pars_, frame_count, fg_abs_index) self._split_VisuFGOrderDesc(visu_pars_, fg_rel_index, frame_count) self._split_VisuCoreSlicePacksDef(visu_pars_) @@ -389,6 +396,6 @@ def _split_VisuCoreSlicePacksSlices(self, visu_pars_, sp_index): def _split_VisuCoreSlicePacksSliceDist(self, visu_pars_, sp_index): VisuCoreSlicePacksSliceDist = visu_pars_['VisuCoreSlicePacksSliceDist'] - value = int(VisuCoreSlicePacksSliceDist.value[sp_index]) + value = int(VisuCoreSlicePacksSliceDist.array[sp_index]) VisuCoreSlicePacksSliceDist.value = value VisuCoreSlicePacksSliceDist.size = 1 From 460744ebd6116230cc7f1a02739063dc046471a3 Mon Sep 17 00:00:00 2001 From: vitous Date: Sat, 31 Jan 2026 01:03:19 +0100 Subject: [PATCH 3/4] enable most of the tests, test on all datasets, migrate datasets into conftest.py --- .github/workflows/python-ci.yml | 70 +++++++---- test/conftest.py | 204 ++++++++++++++++++++++++++++---- test/test_dataset.py | 2 - test/test_jcampdx.py | 10 +- test/test_random_access.py | 33 +++--- test/test_split.py | 36 ++++-- 6 files changed, 274 insertions(+), 81 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 327af66..788aeb4 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -2,22 +2,28 @@ name: CI on: push: - branches: [ master, dev ] + branches: [ master, dev, "dev_JV" ] pull_request: branches: [ master ] jobs: - build: + + # ------------------------------------------------------------ + # 1) INSTALL-ONLY MATRIX + # ------------------------------------------------------------ + install: + name: Install check (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -25,31 +31,49 @@ jobs: - name: Upgrade pip and build tools run: python -m pip install --upgrade pip setuptools wheel build - - name: Install brukerapi with dev dependencies + - name: Install brukerapi (no tests) run: | git clone https://github.com/isi-nmr/brukerapi-python.git cd brukerapi-python - pip install pytest zenodo_get - pip install -e .[dev] --use-pep517 + pip install -e . --use-pep517 - - name: Download test data from Zenodo - working-directory: brukerapi-python/test - run: python -m zenodo_get 10.5281/zenodo.4522220 + # ------------------------------------------------------------ + # 2) FULL TEST JOB + # ------------------------------------------------------------ + test: + name: Full tests (Python 3.13) + runs-on: ubuntu-latest + needs: install - - name: Test using the PV5.1 data set - working-directory: brukerapi-python/test - run: | - unzip 0.2H2.zip - python -m pytest . --test_data "0.2H2" --test_suites="test_data" -v + strategy: + matrix: + python-version: ["3.13"] - - name: Test using the PV6.0.1 data set - working-directory: brukerapi-python/test + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Upgrade pip and build tools + run: python -m pip install --upgrade pip setuptools wheel build + + - name: Install brukerapi with dev dependencies run: | - unzip 20200612_094625_lego_phantom_3_1_2.zip - python -m pytest . --test_data "20200612_094625_lego_phantom_3_1_2" --test_suites="test_data" -v + git clone https://github.com/isi-nmr/brukerapi-python.git + cd brukerapi-python + pip install pytest zenodo_get + pip install -e .[dev] --use-pep517 - - name: Test using the PV7.0.0 data set + - name: Cache Zenodo data + uses: actions/cache@v3 + with: + path: brukerapi-python/test/zenodo_zips + key: zenodo-4522220 + + - name: Run all dataset tests working-directory: brukerapi-python/test - run: | - unzip 20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip - python -m pytest . --test_data "20210128_122257_LEGO_PHANTOM_API_TEST_1_1" --test_suites="test_data" -v \ No newline at end of file + run: python -m pytest . --test_suites="test_data" -v \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index af5c2fd..624334f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,43 +3,191 @@ from pathlib import Path import json from brukerapi.folders import Folder +import subprocess +import zipfile +# ------------------------------- +# Pytest options +# ------------------------------- def pytest_addoption(parser): parser.addoption("--test_data", action="store", default="") parser.addoption("--test_suites", action="store", default="") parser.addoption("--properties_reference", action="store", default="") - # parser.addoption("--properties_reference", action="store", default="") +# ------------------------------- +# Zenodo configuration +# ------------------------------- +ZENODO_DOI = "10.5281/zenodo.4522220" +ZENODO_FILES = { + "PV5.1": "0.2H2.zip", + "PV601": "20200612_094625_lego_phantom_3_1_2.zip", + "PV700": "20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip", +} + +TEST_DIR = Path(__file__).parent +ZENODO_ZIP_DIR = TEST_DIR / "zenodo_zips" +TEST_DATA_ROOT = TEST_DIR / "test_data" + + + +# ------------------------------- +# Helpers +# ------------------------------- +def _resolve_requested_datasets(opt: str | None): + if not opt or opt.lower() == "all": + return list(ZENODO_FILES.keys()) + return [opt] + +def _download_zenodo(): + ZENODO_ZIP_DIR.mkdir(exist_ok=True) + subprocess.run( + ["python", "-m", "zenodo_get", ZENODO_DOI, "-o", str(ZENODO_ZIP_DIR)], + check=True + ) + +def _find_jcampdx_files(dataset_name: str): + """ + Returns a list of tuples (dataset_folder, JCAMPDX_file_path) + """ + files = [] + dataset_root = TEST_DATA_ROOT / dataset_name + if not dataset_root.exists(): + return files + + # Iterate over all dataset subfolders + for subfolder in dataset_root.iterdir(): + if subfolder.is_dir(): + for f in subfolder.rglob("method"): # find all method files recursively + files.append((subfolder, f)) + return files + +def _find_2dseq_datasets(dataset_name: str): + dataset_root = TEST_DATA_ROOT / dataset_name + if not dataset_root.exists(): + return [] + + datasets = [] + for subfolder in dataset_root.iterdir(): + if subfolder.is_dir(): + folder_obj = Folder(subfolder) + for ds in folder_obj.get_dataset_list_rec(): + # Only include if a 2dseq file exists + if ds.type=="2dseq": + datasets.append(ds) + return datasets + +def _ensure_test_data(dataset_name: str): + dataset_dir = TEST_DATA_ROOT / dataset_name + if dataset_dir.exists() and any(dataset_dir.iterdir()): + return + + if dataset_name not in ZENODO_FILES: + raise pytest.UsageError( + f"Unknown test dataset '{dataset_name}'. Available: {', '.join(ZENODO_FILES)}" + ) + + zip_path = ZENODO_ZIP_DIR / ZENODO_FILES[dataset_name] + if not zip_path.exists(): + _download_zenodo() + + dataset_dir.mkdir(parents=True, exist_ok=True) + + # Extract and flatten top-level folder + with zipfile.ZipFile(zip_path) as zf: + top_level_dirs = set() + for member in zf.namelist(): + parts = member.split("/") + if parts[0]: + top_level_dirs.add(parts[0]) + + if len(top_level_dirs) == 1: + top_folder = list(top_level_dirs)[0] + for member in zf.namelist(): + flattened_member = "/".join(member.split("/")[1:]) + if flattened_member: + target_path = dataset_dir / flattened_member + target_path.parent.mkdir(parents=True, exist_ok=True) + with zf.open(member) as src, open(target_path, "wb") as dst: + dst.write(src.read()) + else: + zf.extractall(dataset_dir) +# ------------------------------- +# Parametrization: one test per dataset +# ------------------------------- def pytest_generate_tests(metafunc): - ids, suites, data = get_test_data(metafunc) - if 'test_parameters' in metafunc.fixturenames and 'test_parameters' in suites: - metafunc.parametrize('test_parameters', data, indirect=True, ids=ids) - elif 'test_properties' in metafunc.fixturenames and 'test_properties' in suites: - metafunc.parametrize('test_properties', data, indirect=True, ids=ids) - elif 'test_data' in metafunc.fixturenames and 'test_data' in suites: - metafunc.parametrize('test_data', data, indirect=True, ids=ids) - -def get_test_data(metafunc): - suites = metafunc.config.option.test_suites.split(" ") - study_id = Path(metafunc.config.option.test_data).name - ids = [] - data = [] - - # if properties test configuration exists + requested = _resolve_requested_datasets(metafunc.config.option.test_data or "all") + ref_state = {} if metafunc.config.option.properties_reference and Path(metafunc.config.option.properties_reference).exists(): - with Path(metafunc.config.option.properties_reference).open() as file: - ref_state = json.load(file) - else: - ref_state = {} + with open(metafunc.config.option.properties_reference) as f: + ref_state = json.load(f) + + # ------------------------------- + # JCAMPDX tests + # ------------------------------- + if 'test_jcampdx_data' in metafunc.fixturenames: + jcamp_ids = [] + jcamp_data = [] + for dataset_name in requested: + _ensure_test_data(dataset_name) + for folder, file_path in _find_jcampdx_files(dataset_name): + jcamp_ids.append(f"{dataset_name}/{folder.name}/{file_path.name}") + jcamp_data.append( + ({'parameters': {}, 'path': file_path.relative_to(folder)}, folder) + ) + metafunc.parametrize("test_jcampdx_data", jcamp_data, ids=jcamp_ids) - for dataset in Folder(Path(os.path.expandvars(metafunc.config.option.test_data))).get_dataset_list_rec(): - ids.append(str(dataset.path)) - data.append((dataset.path, ref_state[dataset.id])) if ref_state else data.append((dataset.path, {})) + # ------------------------------- + # Regular dataset tests + # ------------------------------- + if 'test_data' in metafunc.fixturenames: + data_ids = [] + data_items = [] + for dataset_name in requested: + _ensure_test_data(dataset_name) + dataset_root = TEST_DATA_ROOT / dataset_name + for subfolder in dataset_root.iterdir(): + if subfolder.is_dir(): + folder_obj = Folder(subfolder) + for dataset in folder_obj.get_dataset_list_rec(): + data_ids.append(f"{dataset_name}/{dataset.id}") + data_items.append((dataset.path, ref_state.get(dataset.id, {}))) + metafunc.parametrize('test_data', data_items, indirect=True, ids=data_ids) - return ids, suites, data + # ------------------------------- + # Random access tests + # ------------------------------- + if 'test_ra_data' in metafunc.fixturenames: + ra_ids = [] + ra_items = [] + for dataset_name in requested: + _ensure_test_data(dataset_name) + dataset_root = TEST_DATA_ROOT / dataset_name + for subfolder in dataset_root.iterdir(): + if subfolder.is_dir(): + folder_obj = Folder(subfolder) + for dataset in _find_2dseq_datasets(dataset_name): + ra_ids.append(f"{dataset_name}/{dataset.id}") + ra_items.append((dataset.path, ref_state.get(dataset.id, {}))) + metafunc.parametrize('test_ra_data', ra_items, indirect=True, ids=ra_ids) + # ------------------------------- + # Split tests (only 2dseq datasets) + # ------------------------------- + if 'test_split_data' in metafunc.fixturenames: + split_ids = [] + split_items = [] + for dataset_name in requested: + _ensure_test_data(dataset_name) + for ds in _find_2dseq_datasets(dataset_name): + split_ids.append(f"{dataset_name}/{ds.id}") + split_items.append((ds.path, ref_state.get(ds.id, {}))) + metafunc.parametrize('test_split_data', split_items, indirect=True, ids=split_ids) + +# ------------------------------- +# Fixtures +# ------------------------------- @pytest.fixture(autouse=True) def WRITE_TOLERANCE(): return 1.e6 @@ -61,8 +209,12 @@ def test_data(request): @pytest.fixture() def test_jcampdx_data(request): - return None + return request.param @pytest.fixture() def test_split_data(request): - return None + return request.param + +@pytest.fixture() +def test_ra_data(request): + return request.param \ No newline at end of file diff --git a/test/test_dataset.py b/test/test_dataset.py index 2ee4b3e..ec0e7c3 100644 --- a/test/test_dataset.py +++ b/test/test_dataset.py @@ -2,9 +2,7 @@ from brukerapi.schemas import * import numpy as np import json -import os from pathlib import Path -import shutil import pytest data = 0 diff --git a/test/test_jcampdx.py b/test/test_jcampdx.py index f72fdcd..325a6db 100644 --- a/test/test_jcampdx.py +++ b/test/test_jcampdx.py @@ -3,10 +3,13 @@ from pathlib import Path import pytest -@pytest.mark.skip(reason="in progress") +# @pytest.mark.skip(reason="in progress") def test_jcampdx(test_jcampdx_data): + dataset_info, dataset_folder = test_jcampdx_data + jcamp_file_path = dataset_folder / dataset_info['path'] - j = JCAMPDX(Path(test_jcampdx_data[1]) / test_jcampdx_data[0]['path']) + j = JCAMPDX(jcamp_file_path) + for key, ref in test_jcampdx_data[0]['parameters'].items(): parameter_test = j.get_parameter(key) size_test= parameter_test.size @@ -38,3 +41,6 @@ def test_jcampdx(test_jcampdx_data): else: assert value_ref == value_test + + + diff --git a/test/test_random_access.py b/test/test_random_access.py index d2156d8..0078146 100644 --- a/test/test_random_access.py +++ b/test/test_random_access.py @@ -1,28 +1,25 @@ from brukerapi.dataset import Dataset import numpy as np -from pathlib import Path -import pytest -@pytest.mark.skip(reason="in progress") def test_ra(test_ra_data): - d1 = Dataset(Path(test_ra_data[1])/test_ra_data[0]['path']) + d1 = Dataset(test_ra_data[0]) core_index = tuple(slice(None) for i in range(d1.encoded_dim)) - d2 = Dataset(Path(test_ra_data[1])/test_ra_data[0]['path'], random_access=True) - - if "slices" in test_ra_data[0].keys(): - for s in test_ra_data[0]['slices']: - slice_ = json_to_slice(s) - assert np.array_equal(d1.data[slice_], d2.data[slice_]) - else: - # test by single slice - index - for index in np.ndindex(d1.shape[d1.encoded_dim:]): - assert np.array_equal(d1.data[core_index+index], d2.data[core_index+index]) - - # test all possible slices - for slice_ in generate_slices(d1.shape[d1.encoded_dim:]): - assert np.array_equal(d1.data[core_index + slice_], d2.data[core_index + slice_]) + d2 = Dataset(test_ra_data[0], random_access=True) + + # if "slices" in test_ra_data[0].keys(): + # for s in test_ra_data[0]['slices']: + # slice_ = json_to_slice(s) + # assert np.array_equal(d1.data[slice_], d2.data[slice_]) + # else: + # test by single slice - index + for index in np.ndindex(d1.shape[d1.encoded_dim:]): + assert np.array_equal(d1.data[core_index+index], d2.data[core_index+index]) + + # test all possible slices + for slice_ in generate_slices(d1.shape[d1.encoded_dim:]): + assert np.array_equal(d1.data[core_index + slice_], d2.data[core_index + slice_]) def generate_slices(shape): slices = [] diff --git a/test/test_split.py b/test/test_split.py index 792ed72..ea4500c 100644 --- a/test/test_split.py +++ b/test/test_split.py @@ -1,18 +1,34 @@ from brukerapi.dataset import Dataset -from brukerapi.splitters import * +from brukerapi.splitters import SlicePackageSplitter,FrameGroupSplitter import pytest from pathlib import Path -@pytest.mark.skip(reason="in progress") + def test_split(test_split_data, tmp_path): - dataset = Dataset(Path(test_split_data[1]) / test_split_data[0]['path']) + tmp_path/="FG/" + dataset = Dataset(test_split_data[0]) + + if "<{}>".format('FG_ECHO') not in dataset.dim_type: + return + + datasets = FrameGroupSplitter('FG_ECHO').split(dataset, write=True, path_out=tmp_path) + + assert len(datasets) == dataset.shape[dataset.dim_type.index("<{}>".format('FG_ECHO'))] + + +def test_splitSlicePkg(test_split_data, tmp_path): + tmp_path/="Slice/" + dataset = Dataset(test_split_data[0]) + + if "<{}>".format('FG_SLICE') not in dataset.dim_type: + return + if 'VisuCoreSlicePacksSlices' not in dataset: + return + + + datasets = SlicePackageSplitter().split(dataset, write=True, path_out=tmp_path) + - if test_split_data[0]['splitter'] == 'SlicePackage': - SlicePackageSplitter().split(dataset, write=True, path_out=tmp_path) - elif test_split_data[0]['splitter'] == 'FG_ECHO': - FrameGroupSplitter('FG_ECHO').split(dataset, write=True, path_out=tmp_path) + assert len(datasets) == dataset["VisuCoreSlicePacksSlices"].size[0] - for ref in test_split_data[0]['results'].values(): - ds_split = Dataset(tmp_path / ref['path']) - assert ds_split.shape == tuple(ref['shape']) From a5120a9f20251fa83da4c9d5d3a3b86a1ec2104a Mon Sep 17 00:00:00 2001 From: vitous Date: Sat, 31 Jan 2026 01:06:07 +0100 Subject: [PATCH 4/4] Fix tests and cache in tests, formatting in examples --- .github/workflows/python-ci.yml | 5 +- docs/source/conf.py | 1 + examples/read_2dseq.ipynb | 169 ++++++++++++++++---------------- examples/read_fid.ipynb | 165 +++++++++++++++---------------- examples/split_fg_echo.ipynb | 105 ++++++++++---------- examples/split_sp_demo.ipynb | 98 +++++++++--------- test/conftest.py | 108 +++++++++++--------- 7 files changed, 337 insertions(+), 314 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 788aeb4..07f7577 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -71,9 +71,8 @@ jobs: - name: Cache Zenodo data uses: actions/cache@v3 with: - path: brukerapi-python/test/zenodo_zips + path: test/zenodo_zips key: zenodo-4522220 - name: Run all dataset tests - working-directory: brukerapi-python/test - run: python -m pytest . --test_suites="test_data" -v \ No newline at end of file + run: python -m pytest test -v \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index efea38e..b7777c6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,6 +12,7 @@ # import os import sys + sys.path.insert(0, os.path.abspath('../..')) diff --git a/examples/read_2dseq.ipynb b/examples/read_2dseq.ipynb index 8ca59ee..a01e400 100644 --- a/examples/read_2dseq.ipynb +++ b/examples/read_2dseq.ipynb @@ -20,41 +20,48 @@ { "cell_type": "code", "execution_count": 3, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ - "from brukerapi.dataset import Dataset\n", - "from pathlib import Path\n", "import os\n", + "from pathlib import Path\n", + "\n", + "from brukerapi.dataset import Dataset\n", "\n", "# Dataset can be constructed passing a path to the file\n", "# dataset = Dataset(os.environ['PATH_DATA'] / Path('20200612_094625_lego_phantom_3_1_2/5/pdata/1/2dseq'))\n", "\n", "# or to a folder which contains it\n", "dataset = Dataset(os.environ['PATH_DATA'] / Path('20200612_094625_lego_phantom_3_1_2/5/pdata/1'))" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "## Data\n", - "Once a dataset is created, it is possible to access its data, stored as a standard numpy [array](https://numpy.org/doc/stable/reference/generated/numpy.array.html)." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "## Data\n", + "Once a dataset is created, it is possible to access its data, stored as a standard numpy [array](https://numpy.org/doc/stable/reference/generated/numpy.array.html)." + ] }, { "cell_type": "code", "execution_count": 6, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -68,30 +75,30 @@ "source": [ "print(dataset.data.shape)\n", "print(dataset.data.dtype)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "## Parameters\n", - "Values of dataset parameters stored in JCAMP-DX files can be accessed as follows." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "## Parameters\n", + "Values of dataset parameters stored in JCAMP-DX files can be accessed as follows." + ] }, { "cell_type": "code", "execution_count": 11, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -103,55 +110,55 @@ ], "source": [ "print(dataset['VisuAcqSequenceName'].value)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "By default, it is only possible to access parameters stored in JCAMP-DX files related to given dataset type (list of files related to each dataset type can be found in the DEFAULT_STATES variable in the dataset.py file). For instance, a **2dseq** dataset only allows to access parameters from the **visu_pars** file. However, it is possible to construct the dataset, with additional JCAMP-DX files, for instance, it is possible to create a 2dseq dataset while loading parameters from the method file by setting up the `parameter_files` kwarg." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "By default, it is only possible to access parameters stored in JCAMP-DX files related to given dataset type (list of files related to each dataset type can be found in the DEFAULT_STATES variable in the dataset.py file). For instance, a **2dseq** dataset only allows to access parameters from the **visu_pars** file. However, it is possible to construct the dataset, with additional JCAMP-DX files, for instance, it is possible to create a 2dseq dataset while loading parameters from the method file by setting up the `parameter_files` kwarg." + ] }, { "cell_type": "code", "execution_count": 4, - "outputs": [], - "source": [ - "dataset = Dataset(os.environ['PATH_DATA'] / Path('20200612_094625_lego_phantom_3_1_2/5/pdata/1'), parameter_files=['method'])" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "dataset = Dataset(os.environ['PATH_DATA'] / Path('20200612_094625_lego_phantom_3_1_2/5/pdata/1'), parameter_files=['method'])" + ] }, { "cell_type": "markdown", - "source": [ - "Then it is possible to access values of parameters stored in method file.\n" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "Then it is possible to access values of parameters stored in method file.\n" + ] }, { "cell_type": "code", "execution_count": 5, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -163,29 +170,29 @@ ], "source": [ "print(dataset['PVM_AcquisitionTime'].value)\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", - "source": [ - "It is alse possible to add a parameter file to a `Dataset` instance by other parameter files ex post, after it has been created using the `add_parameter_file()` function." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "It is alse possible to add a parameter file to a `Dataset` instance by other parameter files ex post, after it has been created using the `add_parameter_file()` function." + ] }, { "cell_type": "code", "execution_count": 6, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", @@ -198,16 +205,13 @@ "source": [ "dataset.add_parameter_file('reco')\n", "print(dataset['RECO_inp_size'].value)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "## Properties\n", "\n", @@ -215,38 +219,41 @@ "\n", "The `dim_type` property of `Dataset` class can be used in order to make sense out of individual dimensions of the data\n", "array." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "print(dataset.dim_type)\n" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "print(dataset.dim_type)\n" + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Several most wanted properties of MR measurement, such as the echo time, the flip angle, etc. are implemented as\n", "dataset properties." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "print(dataset.id)\n", @@ -254,13 +261,7 @@ "print(dataset.TE)\n", "print(dataset.TR)\n", "print(dataset.imaging_frequency)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] } ], "metadata": { diff --git a/examples/read_fid.ipynb b/examples/read_fid.ipynb index 53da4e4..6e1e2a7 100644 --- a/examples/read_fid.ipynb +++ b/examples/read_fid.ipynb @@ -25,116 +25,124 @@ { "cell_type": "code", "execution_count": 7, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [], "source": [ - "from brukerapi.dataset import Dataset\n", "from pathlib import Path\n", "\n", + "from brukerapi.dataset import Dataset\n", + "\n", "# path to data cloned from\n", "data_path = Path('C:/data/bruker2nifti_qa')\n", "\n", "# both constructors are possible\n", "dataset = Dataset(data_path / Path('raw/Cyceron_DWI/20170719_075627_Lego_1_1/2/fid'))\n", "# dataset = Dataset(data_path / 'raw/Cyceron_DWI/20170719_075627_Lego_1_1/2')" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", - "source": [ - "Once dataset is created, it is possible to access the data array." - ], "metadata": { "collapsed": false - } + }, + "source": [ + "Once dataset is created, it is possible to access the data array." + ] }, { "cell_type": "code", "execution_count": 8, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "(128, 80, 5, 9, 4)\n" - ], - "output_type": "stream" + ] } ], "source": [ "data = dataset.data\n", "print(data.shape)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "The `dim_type` property of `Dataset` class can be used in order to make sense out of individual dimensions of the data \n", "array." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 9, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "['kspace_encode_step_0', 'kspace_encode_step_1', 'slice', 'repetition', 'channel']\n" - ], - "output_type": "stream" + ] } ], "source": [ "print(dataset.dim_type)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Several most wanted properties of MR measurement, such as the echo time, the flip angle, etc. are implemented as \n", "dataset properties." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 10, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%% \n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "34\n", "2000\n", "90\n", "208333.333333333\n", "300.343848550623\n" - ], - "output_type": "stream" + ] } ], "source": [ @@ -143,79 +151,79 @@ "print(dataset.flip_angle)\n", "print(dataset.sw)\n", "print(dataset.transmitter_freq)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% \n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Other parameters can be accessed using the dot notation. It is only possible to access parameters stored in jcamp-dx \n", "files related to given data set type. A **fid** data set only allows to access parameters from the **method** and \n", "**acqp** files." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 11, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "[-16 -12 -8 -4 0 4 8 12 16 20 24 28 32 36 40 44 48 52\n", " 56 60 -15 -11 -7 -3 1 5 9 13 17 21 25 29 33 37 41 45\n", " 49 53 57 61 -14 -10 -6 -2 2 6 10 14 18 22 26 30 34 38\n", " 42 46 50 54 58 62 -13 -9 -5 -1 3 7 11 15 19 23 27 31\n", " 35 39 43 47 51 55 59 63]\n" - ], - "output_type": "stream" + ] } ], "source": [ "# get parameter object\n", "print(dataset.PVM_EncSteps1)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "It is possible to augment `Dataset` by other parameter files, but only files in the same reco, or respective scan \n", "folders are allowed. Please bear in mind, that this augmentation might lead to a loss of uniqueness in the parameter \n", "space.\n", "\n", "You can use it for instance to access reconstruction parameters, or the shape of excitation pulse." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 12, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "[ 0 80]\n", "[[ 1.036651 0. 1.074475 ... 0. 100. 0. ]\n", " [100. 0. 100. ... 0. 1.036651 0. ]]\n" - ], - "output_type": "stream" + ] } ], "source": [ @@ -225,17 +233,10 @@ "print(dataset.XYPOINTS)\n", "\n", "\n", - " \n", + "\n", "\n", "\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] } ], "metadata": { @@ -259,10 +260,10 @@ "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, diff --git a/examples/split_fg_echo.ipynb b/examples/split_fg_echo.ipynb index a51a864..6577067 100644 --- a/examples/split_fg_echo.ipynb +++ b/examples/split_fg_echo.ipynb @@ -27,47 +27,55 @@ { "cell_type": "code", "execution_count": 5, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [], "source": [ - "from brukerapi.dataset import Dataset\n", "from pathlib import Path\n", "\n", + "from brukerapi.dataset import Dataset\n", + "\n", "# path to data cloned from\n", "data_path = Path('C:/data/bruker2nifti_qa')\n", "\n", "# create Dataset\n", "dataset = Dataset(data_path / 'raw/Cyceron_MultiEcho/20170720_080545_Lego_1_2/2/pdata/1/2dseq')\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", - "source": [ - "The 2dseq data set has the following parameters:" - ], "metadata": { "collapsed": false - } + }, + "source": [ + "The 2dseq data set has the following parameters:" + ] }, { "cell_type": "code", "execution_count": 6, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "(128, 128, 2, 9)\n", "['spatial', 'spatial', 'FG_ECHO', 'FG_SLICE']\n", "[[2, '', '<>', 0, 1], [9, '', '<>', 1, 2]]\n", "[12 60]\n" - ], - "output_type": "stream" + ] } ], "source": [ @@ -75,63 +83,64 @@ "print(dataset.dim_type)\n", "print(dataset.VisuFGOrderDesc)\n", "print(dataset.VisuAcqEchoTime)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "## Split\n", "\n", "The FG_ECHO dimension contains 2 echeos, so we can split it into 2 separate 2dseq data sets using \n", "splitter. \n", "\n" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": 7, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [], "source": [ "from brukerapi.splitters import FrameGroupSplitter\n", "\n", "datasets = FrameGroupSplitter('FG_ECHO').split(dataset)\n", "\n", - "# there is the option to save the results as well \n", + "# there is the option to save the results as well\n", "# datasets = FrameGroupSplitter('FG_ECHO').split(dataset, write=True, path_out='tmp')" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", - "source": [ - "Each of the splitted data sets now contains data from the individual frame packages:" - ], "metadata": { "collapsed": false - } + }, + "source": [ + "Each of the splitted data sets now contains data from the individual frame packages:" + ] }, { "cell_type": "code", "execution_count": 8, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "ORIGINAL DATA SET\n", "(128, 128, 2, 9)\n", @@ -145,8 +154,7 @@ "(128, 128, 9)\n", "[9, '', '<>', 1, 2]\n", "60.0\n" - ], - "output_type": "stream" + ] } ], "source": [ @@ -162,14 +170,7 @@ "print(datasets[1].shape)\n", "print(datasets[1].VisuFGOrderDesc)\n", "print(datasets[1].VisuAcqEchoTime)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] } ], "metadata": { @@ -193,10 +194,10 @@ "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, diff --git a/examples/split_sp_demo.ipynb b/examples/split_sp_demo.ipynb index 301389f..62d726a 100644 --- a/examples/split_sp_demo.ipynb +++ b/examples/split_sp_demo.ipynb @@ -2,6 +2,12 @@ "cells": [ { "cell_type": "markdown", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Split 2dseq data set with multiple packages\n", "\n", @@ -17,13 +23,7 @@ "Create a bruker data set from the downloaded data. \n", "\n", "NOTE: It is necessary to change the path to the destination of your bruker2nifti_qa data." - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", @@ -36,10 +36,10 @@ }, "outputs": [], "source": [ - "from brukerapi.dataset import Dataset\n", - "import json\n", "from pathlib import Path\n", "\n", + "from brukerapi.dataset import Dataset\n", + "\n", "# path to data cloned from bruker2nifti_qa\n", "data_path = Path('C:/data/bruker2nifti_qa')\n", "\n", @@ -48,79 +48,87 @@ }, { "cell_type": "markdown", - "source": [ - "The 2dseq data set has the following parameters:" - ], "metadata": { "collapsed": false - } + }, + "source": [ + "The 2dseq data set has the following parameters:" + ] }, { "cell_type": "code", "execution_count": 2, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "(192, 192, 15)\n", "['spatial', 'spatial', 'FG_SLICE']\n" - ], - "output_type": "stream" + ] } ], "source": [ "print(dataset.shape)\n", "print(dataset.dim_type)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", - "source": [ - "The slice dimension contains 3 slice packages, so we can split it into 3 separate 2dseq data sets using splitter. " - ], "metadata": { "collapsed": false - } + }, + "source": [ + "The slice dimension contains 3 slice packages, so we can split it into 3 separate 2dseq data sets using splitter. " + ] }, { "cell_type": "code", "execution_count": 3, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [], "source": [ "from brukerapi.splitters import SlicePackageSplitter\n", "\n", "datasets = SlicePackageSplitter().split(dataset)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] }, { "cell_type": "markdown", - "source": [ - "Each of the splitted data sets now contains data from the individual frame packages:" - ], "metadata": { "collapsed": false - } + }, + "source": [ + "Each of the splitted data sets now contains data from the individual frame packages:" + ] }, { "cell_type": "code", "execution_count": 4, + "metadata": { + "collapsed": false, + "pycharm": { + "is_executing": false, + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", + "output_type": "stream", "text": [ "ORIGINAL DATA SET\n", "(192, 192, 15)\n", @@ -194,8 +202,7 @@ " [ 1 0 0 0 0 -1 0 1 0]\n", " [ 1 0 0 0 0 -1 0 1 0]]\n", "5\n" - ], - "output_type": "stream" + ] } ], "source": [ @@ -224,14 +231,7 @@ "print(datasets[2].VisuCoreFrameCount)\n", "\n", "\n" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n", - "is_executing": false - } - } + ] } ], "metadata": { diff --git a/test/conftest.py b/test/conftest.py index 624334f..a833f42 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,12 @@ -import os -import pytest -from pathlib import Path import json -from brukerapi.folders import Folder import subprocess import zipfile +from pathlib import Path +import sys +import pytest +import os +from brukerapi.folders import Folder + # ------------------------------- # Pytest options @@ -20,7 +22,7 @@ def pytest_addoption(parser): ZENODO_DOI = "10.5281/zenodo.4522220" ZENODO_FILES = { - "PV5.1": "0.2H2.zip", + "PV51": "0.2H2.zip", "PV601": "20200612_094625_lego_phantom_3_1_2.zip", "PV700": "20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip", } @@ -31,6 +33,11 @@ def pytest_addoption(parser): +def pytest_sessionstart(session): + for dataset in ZENODO_FILES: + _ensure_test_data(dataset) + + # ------------------------------- # Helpers # ------------------------------- @@ -40,12 +47,33 @@ def _resolve_requested_datasets(opt: str | None): return [opt] def _download_zenodo(): - ZENODO_ZIP_DIR.mkdir(exist_ok=True) - subprocess.run( - ["python", "-m", "zenodo_get", ZENODO_DOI, "-o", str(ZENODO_ZIP_DIR)], - check=True + ZENODO_ZIP_DIR.mkdir(parents=True, exist_ok=True) + + process = subprocess.Popen( + [ + sys.executable, + "-m", + "zenodo_get", + ZENODO_DOI, + "-o", + str(ZENODO_ZIP_DIR), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # merge stderr into stdout + text=True, + bufsize=1, # line-buffered ) - + + for line in process.stdout: + print(line, end="") # stream live + + returncode = process.wait() + if returncode != 0: + pytest.exit( + f"Zenodo download failed with exit code {returncode}", + returncode=1, + ) + def _find_jcampdx_files(dataset_name: str): """ Returns a list of tuples (dataset_folder, JCAMPDX_file_path) @@ -60,7 +88,7 @@ def _find_jcampdx_files(dataset_name: str): if subfolder.is_dir(): for f in subfolder.rglob("method"): # find all method files recursively files.append((subfolder, f)) - return files + return files def _find_2dseq_datasets(dataset_name: str): dataset_root = TEST_DATA_ROOT / dataset_name @@ -88,30 +116,23 @@ def _ensure_test_data(dataset_name: str): ) zip_path = ZENODO_ZIP_DIR / ZENODO_FILES[dataset_name] - if not zip_path.exists(): + + # Download if missing OR corrupted + if zip_path.exists(): + try: + with zipfile.ZipFile(zip_path, "r") as zf: + bad_file = zf.testzip() + if bad_file is not None: + zip_path.unlink() # corrupted → delete + raise zipfile.BadZipFile + except zipfile.BadZipFile: + _download_zenodo() + else: _download_zenodo() - dataset_dir.mkdir(parents=True, exist_ok=True) - - # Extract and flatten top-level folder - with zipfile.ZipFile(zip_path) as zf: - top_level_dirs = set() - for member in zf.namelist(): - parts = member.split("/") - if parts[0]: - top_level_dirs.add(parts[0]) - - if len(top_level_dirs) == 1: - top_folder = list(top_level_dirs)[0] - for member in zf.namelist(): - flattened_member = "/".join(member.split("/")[1:]) - if flattened_member: - target_path = dataset_dir / flattened_member - target_path.parent.mkdir(parents=True, exist_ok=True) - with zf.open(member) as src, open(target_path, "wb") as dst: - dst.write(src.read()) - else: - zf.extractall(dataset_dir) + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(dataset_dir) + # ------------------------------- # Parametrization: one test per dataset # ------------------------------- @@ -130,7 +151,7 @@ def pytest_generate_tests(metafunc): jcamp_ids = [] jcamp_data = [] for dataset_name in requested: - _ensure_test_data(dataset_name) + for folder, file_path in _find_jcampdx_files(dataset_name): jcamp_ids.append(f"{dataset_name}/{folder.name}/{file_path.name}") jcamp_data.append( @@ -145,7 +166,7 @@ def pytest_generate_tests(metafunc): data_ids = [] data_items = [] for dataset_name in requested: - _ensure_test_data(dataset_name) + dataset_root = TEST_DATA_ROOT / dataset_name for subfolder in dataset_root.iterdir(): if subfolder.is_dir(): @@ -162,7 +183,7 @@ def pytest_generate_tests(metafunc): ra_ids = [] ra_items = [] for dataset_name in requested: - _ensure_test_data(dataset_name) + dataset_root = TEST_DATA_ROOT / dataset_name for subfolder in dataset_root.iterdir(): if subfolder.is_dir(): @@ -179,7 +200,6 @@ def pytest_generate_tests(metafunc): split_ids = [] split_items = [] for dataset_name in requested: - _ensure_test_data(dataset_name) for ds in _find_2dseq_datasets(dataset_name): split_ids.append(f"{dataset_name}/{ds.id}") split_items.append((ds.path, ref_state.get(ds.id, {}))) @@ -192,29 +212,29 @@ def pytest_generate_tests(metafunc): def WRITE_TOLERANCE(): return 1.e6 -@pytest.fixture() +@pytest.fixture def test_parameters(request): return request.param -@pytest.fixture() +@pytest.fixture def test_properties(request): try: return request.param except AttributeError: return None -@pytest.fixture() +@pytest.fixture def test_data(request): return request.param -@pytest.fixture() +@pytest.fixture def test_jcampdx_data(request): return request.param -@pytest.fixture() +@pytest.fixture def test_split_data(request): return request.param -@pytest.fixture() +@pytest.fixture def test_ra_data(request): - return request.param \ No newline at end of file + return request.param