fix(datashare-python): align pagination with spec

.github/workflows/test_asr_worker.yml

@@ -14,9 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check workers/asr-worker
       - name: Lint test

.github/workflows/test_datashare_python.yml

@@ -13,9 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check datashare-python
       - name: Lint test

.github/workflows/test_doc.yml

@@ -13,9 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check docs
       - name: Lint test

.github/workflows/test_extract_worker.yml

@@ -14,9 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check workers/extract-worker
       - name: Lint test

.github/workflows/test_translation_worker.yml

@@ -14,9 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check workers/translation-worker
       - name: Lint test

.github/workflows/test_worker_template.yml

@@ -14,9 +14,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check worker-template
       - name: Lint test

.github/workflows/test_workflows_worker.yml

@@ -16,9 +16,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/ruff-action@v3
+      - uses: astral-sh/ruff-action@v4.1.0
         with:
           args: "--version" # skips test by displaying the version
+          version: "0.16.1"
       - name: Check formatting
         run: ruff format --config qa/ruff.toml --check workers/workflows-worker
       - name: Lint test

datashare-python/datashare_python/cli/worker.py

@@ -78,7 +78,7 @@ async def list_activities(
 
 
 @worker_app.async_command(help=_START_WORKER_HELP)
-async def start(
+async def start(  # noqa: PLR0917
     queue: Annotated[str, typer.Option("--queue", "-q", help=_WORKER_QUEUE_HELP)],
     workflows: Annotated[
         list[str] | None,

datashare-python/datashare_python/objects.py

@@ -317,13 +317,6 @@ class PaginationType(StrEnum):
     BYTE_RANGES = "byteRanges"
 
 
-class BasePagination(DatashareModel, Registrable, ABC):
-    registry_key: ClassVar[str] = Field(frozen=True, default="type")
-
-    total: int
-    type: ClassVar[PaginationType] = Field(frozen=True)
-
-
 def _validate_pages_range(v: Any) -> None:
     if not isinstance(v, list):
         msg = f"expected a list, got {type(v)}"
@@ -345,6 +338,11 @@ def _validate_pages_range(v: Any) -> None:
 PagesRange = Annotated[list[tuple[int, int]], AfterValidator(_validate_pages_range)]
 
 
+class BasePagination(Registrable, DatashareModel):
+    registry_key: ClassVar[str] = Field(frozen=True, default="type")
+    type: ClassVar[PaginationType] = Field(frozen=True)
+
+
 @BasePagination.register(PaginationType.FILESYSTEM)
 class FilesystemPagination(BasePagination):
     type: ClassVar[PaginationType] = Field(
@@ -359,24 +357,33 @@ class ByteRangesPagination(BasePagination):
     )
     byte_ranges: PagesRange
 
+
+pagination_discriminator = make_enum_discriminator("type", PaginationType)
+Pagination = Annotated[
+    tagged_union(BasePagination.__subclasses__(), lambda x: x.type.default),
+    Discriminator(pagination_discriminator),
+]
+
+
+class Pages(DatashareModel):
+    total: int
+    pagination: Pagination
+
     @model_validator(mode="after")
     def byte_ranges_length_should_match_total(self) -> Self:
-        if len(self.byte_ranges) != self.total:
+        if (
+            isinstance(self.pagination, ByteRangesPagination)
+            and len(self.pagination.byte_ranges) != self.total
+        ):
+            n_pages = len(self.pagination.byte_ranges)
             msg = (
-                f"byte_ranges must match total. Found {len(self.byte_ranges)} for"
+                f"byte_ranges must match total. Found {n_pages} for"
                 f" byte_ranges and  {self.total} for total."
             )
             raise ValueError(msg)
         return self
 
 
-pagination_discriminator = make_enum_discriminator("type", PaginationType)
-Pagination = Annotated[
-    tagged_union(BasePagination.__subclasses__(), lambda x: x.type.default),
-    Discriminator(pagination_discriminator),
-]
-
-
 class DocArtifact(BaseModel, ABC):
     # This object is not used for serde, just as a container, it's OK to allow
     # arbitrary types (to allow storing BytesIO)

datashare-python/pyproject.toml

@@ -72,7 +72,7 @@ dev = [
   "pytest~=8.1",
   "pytest-asyncio~=0.24",
   "redis[hiredis]~=5.2.1",
-  "ruff==0.15.2",
+  "ruff==0.16.1",
   "typing-extensions~=4.15.0",
 ]
 

datashare-python/tests/test_objects.py

@@ -7,12 +7,13 @@
 from datashare_python.conftest import TEST_PROJECT
 from datashare_python.constants import TIKA_METADATA_RESOURCENAME
 from datashare_python.objects import (
-    BasePagination,
     ByteRangesPagination,
     DatashareLanguage,
     Document,
     DocumentLocation,
     FilesystemDocument,
+    FilesystemPagination,
+    Pages,
     Task,
     TaskState,
 )
@@ -99,18 +100,42 @@ def test_invalid_datashare_language_should_raise(
         type_adapter.validate_python(language)
 
 
-def test_pagination_serde() -> None:
-    # Given
-    pagination = ByteRangesPagination(total=3, byte_ranges=[(0, 1), (1, 2), (2, 3)])
-    ta = TypeAdapter(BasePagination)
+@pytest.mark.parametrize(
+    ("pages", "expected_serialized"),
+    [
+        (
+            Pages(
+                pagination=ByteRangesPagination(byte_ranges=[(0, 1), (1, 2), (2, 3)]),
+                total=3,
+            ),
+            {
+                "pagination": {
+                    "byteRanges": [[0, 1], [1, 2], [2, 3]],
+                    "type": "byteRanges",
+                },
+                "total": 3,
+            },
+        ),
+        (
+            Pages(pagination=FilesystemPagination(), total=3),
+            {"pagination": {"type": "filesystem"}, "total": 3},
+        ),
+    ],
+)
+def test_pages_serde(pages: Pages, expected_serialized: dict) -> None:
     # When
-    serialized = pagination.model_dump_json(by_alias=True)
-    deserialized = ta.validate_json(serialized)
+    serialized = pages.model_dump_json(by_alias=True)
+    deserialized = Pages.model_validate_json(serialized)
     # Then
-    expected_serialized = {
-        "type": "byteRanges",
-        "total": 3,
-        "byteRanges": [[0, 1], [1, 2], [2, 3]],
-    }
     assert json.loads(serialized) == expected_serialized
-    assert deserialized == pagination
+    assert deserialized == pages
+
+
+def test_pages_validation_should_raise_for_inconsistent_byte_ranges() -> None:
+    # When
+    expected = "byte_ranges must match total"
+    with pytest.raises(ValidationError, match=expected):
+        Pages(
+            pagination=ByteRangesPagination(byte_ranges=[(0, 1), (1, 2), (2, 3)]),
+            total=2,
+        )