Enable most of the tests, Fix errors in splitters. Add installation tests for all supported python versions. Enable data cache in tests. Add __contains__ on dataset and jcampdx

.github/workflows/python-ci.yml

@@ -2,54 +2,77 @@ name: CI
 
 on:
   push:
-    branches: [ master, dev ]
+    branches: [ master, dev, "dev_JV" ]
   pull_request:
     branches: [ master ]
 
 jobs:
-  build:
+
+  # ------------------------------------------------------------
+  # 1) INSTALL-ONLY MATRIX
+  # ------------------------------------------------------------
+  install:
+    name: Install check (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-         python-version: ["3.13"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
 
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Upgrade pip and build tools
       run: python -m pip install --upgrade pip setuptools wheel build
 
-    - name: Install brukerapi with dev dependencies
+    - name: Install brukerapi (no tests)
       run: |
         git clone https://github.com/isi-nmr/brukerapi-python.git
         cd brukerapi-python
-        pip install pytest zenodo_get
-        pip install -e .[dev] --use-pep517
+        pip install -e . --use-pep517
 
-    - name: Download test data from Zenodo
-      working-directory: brukerapi-python/test
-      run: python -m zenodo_get 10.5281/zenodo.4522220
+  # ------------------------------------------------------------
+  # 2) FULL TEST JOB
+  # ------------------------------------------------------------
+  test:
+    name: Full tests (Python 3.13)
+    runs-on: ubuntu-latest
+    needs: install
 
-    - name: Test using the PV5.1 data set
-      working-directory: brukerapi-python/test
-      run: |
-        unzip 0.2H2.zip
-        python -m pytest . --test_data "0.2H2" --test_suites="test_data" -v
+    strategy:
+      matrix:
+        python-version: ["3.13"]
 
-    - name: Test using the PV6.0.1 data set
-      working-directory: brukerapi-python/test
-      run: |
-        unzip 20200612_094625_lego_phantom_3_1_2.zip
-        python -m pytest . --test_data "20200612_094625_lego_phantom_3_1_2" --test_suites="test_data" -v
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-    - name: Test using the PV7.0.0 data set
-      working-directory: brukerapi-python/test
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Upgrade pip and build tools
+      run: python -m pip install --upgrade pip setuptools wheel build
+
+    - name: Install brukerapi with dev dependencies
       run: |
-        unzip 20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip
-        python -m pytest . --test_data "20210128_122257_LEGO_PHANTOM_API_TEST_1_1" --test_suites="test_data" -v
+        git clone https://github.com/isi-nmr/brukerapi-python.git
+        cd brukerapi-python
+        pip install pytest zenodo_get
+        pip install -e .[dev] --use-pep517
+
+    - name: Cache Zenodo data
+      uses: actions/cache@v3
+      with:
+        path: test/zenodo_zips
+        key: zenodo-4522220
+
+    - name: Run all dataset tests
+      run: python -m pytest test -v

brukerapi/dataset.py

@@ -166,7 +166,7 @@ def __init__(self, path, **state):
             raise FileNotFoundError(self.path)
 
         # directory constructor
-        if self.path.is_dir():
+        if self.path.is_dir() and state.get('load'):
             content = os.listdir(self.path)
             if 'fid' in content:
                 self.path = self.path / 'fid'
@@ -214,6 +214,14 @@ def __getitem__(self, item):
 
         raise KeyError(item)
 
+
+    def __contains__(self, item):
+        for parameter_file in self._parameters.values():
+            if item in parameter_file:
+                return True
+        return False
+
+
     def __call__(self, **kwargs):
         self._set_state(kwargs)
         return self
@@ -593,7 +601,7 @@ def write(self, path, **kwargs):
         parent = path.parent
 
         if not parent.exists():
-            os.mkdir(parent)
+            os.makedirs(parent,exist_ok=True)
 
         self._write_parameters(parent)
         self._write_data(path)

brukerapi/jcampdx.py

@@ -633,6 +633,12 @@ def __add__(self, other):
     def __getitem__(self, key):
         return self.params[key]
 
+
+    def __contains__(self, item):
+        if item in self.params:
+            return True
+        return False
+
     def __delitem__(self, key):
         del self.params[key]
 

brukerapi/splitters.py

@@ -1,9 +1,10 @@
 from .utils import index_to_slice
 from .dataset import Dataset
-
+import os
 import numpy as np
 import copy
 from pathlib import Path
+from .exceptions import MissingProperty
 
 SUPPORTED_FG = ['FG_ISA','FG_IRMODE','FG_ECHO']
 
@@ -40,7 +41,7 @@ def _split_VisuCoreDataMin(self, dataset, visu_pars, select, fg_rel_index):
         VisuCoreDataMin = visu_pars['VisuCoreDataMin']
         value = np.reshape(VisuCoreDataMin.value, dataset.shape_final[dataset.encoded_dim:], order='F')
         value = value[index_to_slice(select, value.shape, fg_rel_index)]
-        VisuCoreDataMin.size = (np.prod(value.shape),)
+        VisuCoreDataMin.size = (int(np.prod(value.shape)),)
         VisuCoreDataMin.value = value.flatten(order='F')
 
     def _split_VisuCoreDataMax(self, dataset, visu_pars, select, fg_rel_index):
@@ -54,7 +55,7 @@ def _split_VisuCoreDataMax(self, dataset, visu_pars, select, fg_rel_index):
         VisuCoreDataMax = visu_pars['VisuCoreDataMax']
         value = np.reshape(VisuCoreDataMax.value, dataset.shape_final[dataset.encoded_dim:], order='F')
         value = value[index_to_slice(select, value.shape, fg_rel_index)]
-        VisuCoreDataMax.size = (np.prod(value.shape),)
+        VisuCoreDataMax.size = (int(np.prod(value.shape)),)
         VisuCoreDataMax.value = value.flatten(order='F')
 
     def _split_VisuCoreDataOffs(self, dataset, visu_pars, select, fg_rel_index):
@@ -68,7 +69,7 @@ def _split_VisuCoreDataOffs(self, dataset, visu_pars, select, fg_rel_index):
         VisuCoreDataOffs = visu_pars['VisuCoreDataOffs']
         value = np.reshape(VisuCoreDataOffs.value, dataset.shape_final[dataset.encoded_dim:],order='F')
         value = value[index_to_slice(select, value.shape, fg_rel_index)]
-        VisuCoreDataOffs.size = (np.prod(value.shape),)
+        VisuCoreDataOffs.size = (int(np.prod(value.shape)),)
         VisuCoreDataOffs.value = value.flatten(order='F')
 
     def _split_VisuCoreDataSlope(self, dataset, visu_pars, select, fg_rel_index):
@@ -82,7 +83,7 @@ def _split_VisuCoreDataSlope(self, dataset, visu_pars, select, fg_rel_index):
         VisuCoreDataSlope = visu_pars['VisuCoreDataSlope']
         value = np.reshape(VisuCoreDataSlope.value, dataset.shape_final[dataset.encoded_dim:],order='F')
         value = value[index_to_slice(select, value.shape, fg_rel_index)]
-        VisuCoreDataSlope.size = (np.prod(value.shape),)
+        VisuCoreDataSlope.size = (int(np.prod(value.shape)),)
         VisuCoreDataSlope.value = value.flatten(order='F')
 
     def _split_VisuCoreTransposition(self, dataset, visu_pars, index, fg_index):
@@ -93,7 +94,7 @@ def _split_VisuCoreTransposition(self, dataset, visu_pars, index, fg_index):
 
         value = np.reshape(VisuCoreTransposition.value, dataset.shape_final[dataset.encoded_dim:], order='F')
         value = value[index_to_slice(index, value.shape, fg_index - dataset.encoded_dim)]
-        VisuCoreTransposition.size = (np.prod(value.shape),)
+        VisuCoreTransposition.size = (int(np.prod(value.shape)),)
         VisuCoreTransposition.value = value.flatten(order='F')
 
 
@@ -154,9 +155,12 @@ def split(self, dataset, select=None, write=False, path_out=None, **kwargs):
         for select_ in select:
             # construct a new Dataset, without loading data, the data will be supplied later
             name = '{}_{}_{}/2dseq'.format(dataset.path.parents[0].name, self.fg, select_)
-
+
+            dset_path = dataset.path.parents[1] / name
+            os.makedirs(dset_path,exist_ok=True)
+
             # construct a new Dataset, without loading data, the data will be supplied later
-            dataset_ = Dataset(dataset.path.parents[1] / name, load=False)
+            dataset_ = Dataset(dataset.path.parents[1] / name, load=0)
 
             dataset_.parameters = self._split_params(dataset, select_, fg_abs_index, fg_rel_index, fg_size)
 
@@ -250,7 +254,7 @@ def _split_VisuFGElemComment(self, visu_pars, fg_scheme, index, fg_index):
 
     def _split_VisuAcqEchoTime(self, visu_pars, select):
         VisuAcqEchoTime = visu_pars['VisuAcqEchoTime']
-        value = VisuAcqEchoTime.value
+        value = VisuAcqEchoTime.list
         VisuAcqEchoTime.size=(1,)
         VisuAcqEchoTime.value = float(value[select])
 
@@ -272,9 +276,10 @@ def split(self, dataset, write=False, path_out=None):
         """
 
         try:
-            VisuCoreSlicePacksSlices = dataset['VisuCoreSlicePacksSlices'].value
+            VisuCoreSlicePacksSlices = dataset['VisuCoreSlicePacksSlices'].nested
         except KeyError:
-            print('Parameter VisuCoreSlicePacksSlices not found')
+            raise MissingProperty('Parameter VisuCoreSlicePacksSlices not found')
+
 
         # list of split data sets
         datasets = []
@@ -299,9 +304,10 @@ def split(self, dataset, write=False, path_out=None):
             # name of the data set created by the split
             name = '{}_sp_{}/2dseq'.format(dataset.path.parents[0].name, sp_index)
 
+            os.makedirs(dataset.path.parents[1] / name,exist_ok=True)
 
             # construct a new Dataset, without loading data, the data will be supplied later
-            dataset_ = Dataset(dataset.path.parents[1] / name, load=False)
+            dataset_ = Dataset(dataset.path.parents[1] / name, load=0)
 
             # SPLIT parameters
             dataset_.parameters = self._split_parameters(dataset, frame_range, fg_rel_index, fg_abs_index, sp_index, frame_count)
@@ -337,7 +343,8 @@ def _split_parameters(self, dataset, frame_range, fg_rel_index, fg_abs_index, sp
         self._split_VisuCoreDataMax(dataset, visu_pars_, frame_range, fg_rel_index)
         self._split_VisuCoreDataOffs(dataset, visu_pars_, frame_range, fg_rel_index)
         self._split_VisuCoreDataSlope(dataset, visu_pars_, frame_range, fg_rel_index)
-        self._split_VisuCoreTransposition(dataset, visu_pars_, frame_range, fg_rel_index)
+        if "VisuCoreTransposition" in dataset:
+            self._split_VisuCoreTransposition(dataset, visu_pars_, frame_range, fg_rel_index)
         self._split_VisuCoreFrameCount(dataset, visu_pars_, frame_count, fg_abs_index)
         self._split_VisuFGOrderDesc(visu_pars_, fg_rel_index, frame_count)
         self._split_VisuCoreSlicePacksDef(visu_pars_)
@@ -389,6 +396,6 @@ def _split_VisuCoreSlicePacksSlices(self, visu_pars_, sp_index):
 
     def _split_VisuCoreSlicePacksSliceDist(self, visu_pars_, sp_index):
         VisuCoreSlicePacksSliceDist = visu_pars_['VisuCoreSlicePacksSliceDist']
-        value = int(VisuCoreSlicePacksSliceDist.value[sp_index])
+        value = int(VisuCoreSlicePacksSliceDist.array[sp_index])
         VisuCoreSlicePacksSliceDist.value = value
         VisuCoreSlicePacksSliceDist.size = 1

docs/source/conf.py

@@ -12,6 +12,7 @@
 #
 import os
 import sys
+
 sys.path.insert(0, os.path.abspath('../..'))