Major refactor

.github/workflows/docs.yaml

@@ -28,19 +28,22 @@ jobs:
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: environment.yml
-          create-args: python=3.10
+          create-args: python=3.12
           init-shell: bash
           cache-downloads: true
           cache-environment: true
           post-cleanup: all
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - name: Install plinder
+        shell: bash -el {0}
+        run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact --extra docs
       - name: Build docs
         shell: bash -el {0}
         id: build
-        run: |
-          which python
-          python -m pip install six
-          python -m pip install '.[docs,loader]'
-          sphinx-build docs build/docs
+        run: sphinx-build docs build/docs
       - name: Upload build artifact
         uses: actions/upload-pages-artifact@v3
         with:

.github/workflows/main.yaml

@@ -2,7 +2,6 @@ name: Package
 permissions:
   contents: write
   id-token: write
-  packages: write
 on:
   push:
     branches:
@@ -14,51 +13,48 @@ jobs:
     name: Package
     if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip ci]') }}
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Setup python
-        uses: actions/setup-python@v5
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          python-version: "3.10"
-      - name: Configure docker
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
-      - name: Install build and tag requirements
-        run: python -m pip install build semver
+          environment-file: environment.yml
+          create-args: python=3.12
+          init-shell: bash
+          cache-downloads: true
+          cache-environment: true
+          post-cleanup: all
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - name: Install plinder
+        run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact
       - name: Get version bump
         id: get-tag
-        run: echo "bump=$(python flows/docker.py bump)" >> $GITHUB_OUTPUT
-      - name: Pull, build, tag, bump base image
-        if: steps.get-tag.outputs.bump != ''
-        run: python flows/docker.py pull --build --promote
-      - name: Build image
+        run: echo "bump=$(python version.py)" >> $GITHUB_OUTPUT
+      - name: Run tests
         if: steps.get-tag.outputs.bump != ''
-        run: python flows/docker.py build
-      - name: Run tests against new image and save images
+        env:
+          PLINDER_OFFLINE: "true"
+        run: python -m pytest -n auto -v
+      - name: Tag release
         if: steps.get-tag.outputs.bump != ''
-        run: |
-          python flows/docker.py test --push
-          if ! test -f reports/.coverage; then
-            echo "Missing test coverage reports (reports/.coverage). Not all tests have passed?"
-            exit 1
-          else
-            echo "Test coverage report found!"
-          fi
+        run: git tag ${{ steps.get-tag.outputs.bump }}
       - name: Build distribution
-        run: python -m build
+        run: uv build
       - name: Publish package distributions to PyPI
         if: steps.get-tag.outputs.bump != ''
         uses: pypa/gh-action-pypi-publish@release/v1
       - name: Save git tag
         if: steps.get-tag.outputs.bump != ''
         run: git push origin ${{ steps.get-tag.outputs.bump }}
-      - name: Copy and surgery coverage
-        if: steps.get-tag.outputs.bump != ''
-        run: |
-          cp reports/.coverage .
-          sqlite3 .coverage "update file set path='src/' || substr(path, 40);"
       - name: Post coverage comment
         if: steps.get-tag.outputs.bump != ''
         uses: py-cov-action/python-coverage-comment-action@v3

.github/workflows/pr.yaml

@@ -3,7 +3,6 @@ name: CI Workflow
 permissions:
   contents: write
   id-token: write
-  packages: read
   pull-requests: write
 
 concurrency:
@@ -23,67 +22,51 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install tox
         run: pip install tox
       - name: Run quality checks
-        run: tox -e py310-lint,py310-type
+        run: tox -e py312-lint,py312-type
       - name: Directory Cache
         uses: actions/cache@v4
         with:
           path: .tox
-          key: tox-${{ runner.os }}-3.10-${{ hashFiles('tox.ini') }}
+          key: tox-${{ runner.os }}-3.12-${{ hashFiles('tox.ini') }}
           restore-keys: |
-            tox-${{ runner.os }}-3.10-
+            tox-${{ runner.os }}-3.12-
 
   test:
-    name: Build and test docker image
+    name: Test
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
-      - name: Configure docker
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
-      - name: Get changed source files
-        id: changed-src
-        uses: tj-actions/changed-files@v41
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          files: src/**
-      - name: Get changed base image files
-        if: steps.changed-src.outputs.any_changed == 'true'
-        id: changed-base
-        uses: tj-actions/changed-files@v41
+          environment-file: environment.yml
+          create-args: python=3.12
+          init-shell: bash
+          cache-downloads: true
+          cache-environment: true
+          post-cleanup: all
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
         with:
-          files: dockerfiles/base/*
-      - name: Pull base image and build
-        if: steps.changed-src.outputs.any_changed == 'true' && steps.changed-base.outputs.any_changed == 'true'
-        run: python flows/docker.py pull --build
-      - name: Pull base image
-        if: steps.changed-src.outputs.any_changed == 'true' && steps.changed-base.outputs.any_changed != 'true'
-        run: python flows/docker.py pull
-      - name: Build image
-        if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true'
-        run: python flows/docker.py build
-      - name: Run test image
-        if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true'
-        run: |
-          python flows/docker.py test
-          if ! test -f reports/.coverage; then
-            echo "Missing test coverage reports (reports/.coverage). Not all tests have passed?"
-            exit 1
-          else
-            echo "Test coverage report found!"
-          fi
-      - name: Copy and surgery coverage
-        if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true'
-        run: |
-          cp reports/.coverage .
-          sqlite3 .coverage "update file set path='src/' || substr(path, 40);"
+          enable-cache: true
+      - name: Install plinder
+        run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact
+      - name: Run tests
+        env:
+          PLINDER_OFFLINE: "true"
+        run: python -m pytest -n auto -v
       - name: Post coverage comment
-        if: steps.changed-src.outputs.any_changed == 'true'
         uses: py-cov-action/python-coverage-comment-action@v3
         with:
           GITHUB_TOKEN: ${{ github.token }}
@@ -101,12 +84,19 @@ jobs:
         uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: environment.yml
-          create-args: python=3.10
+          create-args: python=3.12
           init-shell: bash
           cache-downloads: true
           cache-environment: true
           post-cleanup: all
-      - name: Install Python requirements
-        run: pip install six ".[docs,loader]"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - name: Install plinder
+        run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact --extra docs
       - name: Build the documentation
         run: sphinx-build --fail-on-warning docs build/docs
+      - name: Show notebook errors
+        if: failure()
+        run: cat build/docs/reports/examples/*.err.log 2>/dev/null || true

.gitignore

@@ -149,6 +149,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.claude
 
 # Spyder project settings
 .spyderproject
@@ -173,7 +174,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.metaflow/
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
@@ -186,9 +186,14 @@ cython_debug/
 */*/DS_Store
 
 src/plinder-data/plinder/data/artifacts
-#src/plinder-core
 *.bak*
 *.1.*
 tests/xx
 tests/test_data/plinder/mount/systems/*/
+tmp_foldseek
+tmp_mmseqs
 artifacts
+logs/
+
+# uv lockfile: not committed; CI resolves the latest versions allowed by pyproject.toml
+uv.lock

.pre-commit-config.yaml

@@ -4,7 +4,6 @@ exclude: |
     (?x)^(
         docs/.*png|
         tests/test_data/.*|
-        src/plinder/data/utils/annotations/static_files/dates.csv|
         private/.*ipynb
     )$
 repos:
@@ -15,15 +14,12 @@ repos:
     - id: end-of-file-fixer
     - id: check-yaml
     - id: check-added-large-files
--   repo: local
+# pinned ruff, so the hook does not depend on whichever ruff is first on PATH
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.4
     hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: bash -c 'ruff format --force-exclude --preview src tests'
-        language: system
-        types: [python]
-      - id: ruff-linter
-        name: ruff-linter
-        entry: bash -c 'ruff check --fix src tests'
-        language: system
-        types: [python]
+    - id: ruff-format
+      files: ^(src|tests)/
+    - id: ruff
+      args: [--fix]
+      files: ^(src|tests)/

.python-version

@@ -0,0 +1 @@
+3.12