Fix PV-360 loading add new datasets

.github/workflows/python-ci.yml

@@ -74,6 +74,12 @@ jobs:
         path: test/zenodo_zips
         key: zenodo-4522220
 
+    - name: Cache PV360 standard data
+      uses: actions/cache@v5
+      with:
+        path: test/test_data/PV360_StdData
+        key: pv360-stddata-6f1b67e5dbc3d7b3646a6315959ccf6d4bd02237
+
     - name: Run all dataset tests
       run: |
         python -m pytest test -v --cov=brukerapi --cov-branch --cov-report=xml --cov-report=term-missing --cov-report=html

brukerapi/config/properties_rawdata_core.json

@@ -112,6 +112,12 @@
     }
   ],
   "shape_storage": [
+    {
+      "cmd": "(@job_desc[0],) + (#PVM_EncNReceivers,) + (@job_desc[6],)",
+      "conditions": [
+        "#ACQ_sw_version.value.startswith('<PV-360.3.')"
+      ]
+    },
     {
       "cmd": "(@job_desc[0],) + (#PVM_EncNReceivers,) + (@job_desc[3],)",
       "conditions": []

brukerapi/schemas.py

@@ -439,8 +439,8 @@ class SchemaRawdata(Schema):
     @property
     def layouts(self):
         layouts = {}
-        layouts["raw"] = (int(self._dataset.job_desc[0] / 2), self._dataset.channels, int(self._dataset.job_desc[3]))
-        layouts["shape_storage"] = (int(self._dataset.job_desc[0]), self._dataset.channels, int(self._dataset.job_desc[3]))
+        layouts["raw"] = (int(self._dataset.shape_storage[0] / 2), self._dataset.channels, int(self._dataset.shape_storage[2]))
+        layouts["shape_storage"] = self._dataset.shape_storage
         layouts["final"] = layouts["raw"]
         return layouts
 

test/conftest.py

@@ -1,6 +1,10 @@
 import json
+import os
+import shutil
 import subprocess
 import sys
+import urllib.parse
+import urllib.request
 import zipfile
 from pathlib import Path
 
@@ -27,7 +31,16 @@ def pytest_addoption(parser):
     "PV51": "0.2H2.zip",
     "PV601": "20200612_094625_lego_phantom_3_1_2.zip",
     "PV700": "20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip",
-    # "PV360V37": "20210128_122257_LEGO_PHANTOM_API_TEST_1_1.zip",
+}
+
+LOCAL_DATASETS = ["PV360-V37"]
+
+GITHUB_DATASETS = {
+    "PV360_StdData": {
+        "repository": "https://github.com/cecilyen/PV360_StdData.git",
+        "revision": "6f1b67e5dbc3d7b3646a6315959ccf6d4bd02237",
+        "media": "https://media.githubusercontent.com/media/cecilyen/PV360_StdData/6f1b67e5dbc3d7b3646a6315959ccf6d4bd02237",
+    },
 }
 
 TEST_DIR = Path(__file__).parent
@@ -38,17 +51,92 @@ def pytest_addoption(parser):
 def pytest_sessionstart(session):
     for dataset in ZENODO_FILES:
         _ensure_test_data(dataset)
+    for dataset in GITHUB_DATASETS:
+        _ensure_github_test_data(dataset)
 
 
 # -------------------------------
 # Helpers
 # -------------------------------
 def _resolve_requested_datasets(opt: str | None):
     if not opt or opt.lower() == "all":
-        return list(ZENODO_FILES.keys())
+        available_local_datasets = [name for name in LOCAL_DATASETS if (TEST_DATA_ROOT / name).is_dir()]
+        return [*ZENODO_FILES, *GITHUB_DATASETS, *available_local_datasets]
     return [opt]
 
 
+def _is_required_github_data_file(path: Path):
+    return path.name in {"2dseq", "traj"} or path.name.startswith("rawdata.job")
+
+
+def _is_git_lfs_pointer(path: Path):
+    try:
+        with path.open("rb") as file:
+            return file.read(42) == b"version https://git-lfs.github.com/spec/v1"
+    except OSError:
+        return False
+
+
+def _download_github_lfs_file(dataset_name: str, path: Path):
+    relative_path = path.relative_to(TEST_DATA_ROOT / dataset_name)
+    media_root = GITHUB_DATASETS[dataset_name]["media"]
+    url = f"{media_root}/{urllib.parse.quote(relative_path.as_posix())}"
+    temporary_path = path.with_name(f"{path.name}.download")
+
+    try:
+        with urllib.request.urlopen(url) as response, temporary_path.open("wb") as output:
+            shutil.copyfileobj(response, output)
+        temporary_path.replace(path)
+    except OSError as error:
+        temporary_path.unlink(missing_ok=True)
+        pytest.exit(f"GitHub test-data download failed for {relative_path}: {error}", returncode=1)
+
+
+def _ensure_github_test_data(dataset_name: str):
+    dataset_dir = TEST_DATA_ROOT / dataset_name
+    config = GITHUB_DATASETS[dataset_name]
+    git_environment = {**os.environ, "GIT_LFS_SKIP_SMUDGE": "1"}
+
+    if not dataset_dir.exists():
+        process = subprocess.run(
+            ["git", "clone", "--depth", "1", config["repository"], str(dataset_dir)],
+            check=False,
+            capture_output=True,
+            text=True,
+            env=git_environment,
+        )
+        if process.returncode != 0:
+            pytest.exit(f"GitHub test-data clone failed: {process.stderr.strip()}", returncode=1)
+
+    revision = subprocess.run(
+        ["git", "-C", str(dataset_dir), "rev-parse", "HEAD"],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if revision.returncode != 0 or revision.stdout.strip() != config["revision"]:
+        fetch = subprocess.run(
+            ["git", "-C", str(dataset_dir), "fetch", "--depth", "1", "origin", config["revision"]],
+            check=False,
+            capture_output=True,
+            text=True,
+            env=git_environment,
+        )
+        checkout = subprocess.run(
+            ["git", "-C", str(dataset_dir), "checkout", "--detach", config["revision"]],
+            check=False,
+            capture_output=True,
+            text=True,
+            env=git_environment,
+        )
+        if fetch.returncode != 0 or checkout.returncode != 0:
+            pytest.exit(f"GitHub test-data checkout failed: {fetch.stderr}{checkout.stderr}".strip(), returncode=1)
+
+    for path in dataset_dir.rglob("*"):
+        if path.is_file() and _is_required_github_data_file(path) and _is_git_lfs_pointer(path):
+            _download_github_lfs_file(dataset_name, path)
+
+
 def _download_zenodo():
     ZENODO_ZIP_DIR.mkdir(parents=True, exist_ok=True)
 

test/test_property_configs.py

@@ -55,6 +55,10 @@ def test_rawdata_pv360_v3_uses_prefix_matching():
         assert "#ACQ_sw_version=='<PV-360.1.1>' or #ACQ_sw_version.value.startswith('<PV-360.3.')" in branch["conditions"]
 
     assert config["job_desc"][1]["conditions"] == ["#ACQ_sw_version.value.startswith('<PV-360.3.')"]
+    assert config["shape_storage"][0] == {
+        "cmd": "(@job_desc[0],) + (#PVM_EncNReceivers,) + (@job_desc[6],)",
+        "conditions": ["#ACQ_sw_version.value.startswith('<PV-360.3.')"],
+    }
 
 
 def test_traj_scheme_detection_is_not_version_gated():

test/test_rawdata.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import pytest
+
+from brukerapi.dataset import Dataset
+
+RAWDATA_JOB_PATHS = sorted(Path("test/test_data").rglob("rawdata.job*"))
+
+
+@pytest.mark.parametrize("rawdata_path", RAWDATA_JOB_PATHS, ids=[str(path) for path in RAWDATA_JOB_PATHS])
+def test_rawdata_job_loads_directly(rawdata_path):
+    dataset = Dataset(rawdata_path)
+
+    assert dataset.type == "rawdata"
+    assert dataset.subtype == rawdata_path.suffix.removeprefix(".")
+    assert dataset.data.shape == dataset._schema.layouts["raw"]
+    assert dataset.data.size > 0