diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index aed210a61..f6edeac6c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -33,14 +33,17 @@ jobs: cache-downloads: true cache-environment: true post-cleanup: all + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - name: Install plinder + shell: bash -el {0} + run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact --extra docs - name: Build docs shell: bash -el {0} id: build - run: | - which python - python -m pip install six - python -m pip install '.[docs,loader]' - sphinx-build docs build/docs + run: sphinx-build docs build/docs - name: Upload build artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cf0e9405c..9ef3df7b9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -2,7 +2,6 @@ name: Package permissions: contents: write id-token: write - packages: write on: push: branches: @@ -14,51 +13,48 @@ jobs: name: Package if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip ci]') }} runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Setup python - uses: actions/setup-python@v5 + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 with: - python-version: "3.12" - - name: Configure docker - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin - - name: Install build and tag requirements - run: python -m pip install build semver + environment-file: environment.yml + create-args: python=3.12 + init-shell: bash + cache-downloads: true + cache-environment: true + post-cleanup: all + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - name: Install plinder + run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact - name: Get version bump id: get-tag - run: echo "bump=$(python flows/docker.py bump)" >> $GITHUB_OUTPUT - - name: Pull, build, tag, bump base image - if: steps.get-tag.outputs.bump != '' - run: python flows/docker.py pull --build --promote - - name: Build image + run: echo "bump=$(python version.py)" >> $GITHUB_OUTPUT + - name: Run tests if: steps.get-tag.outputs.bump != '' - run: python flows/docker.py build - - name: Run tests against new image and save images + env: + PLINDER_OFFLINE: "true" + run: python -m pytest -n auto -v + - name: Tag release if: steps.get-tag.outputs.bump != '' - run: | - python flows/docker.py test --push - if ! test -f reports/.coverage; then - echo "Missing test coverage reports (reports/.coverage). Not all tests have passed?" - exit 1 - else - echo "Test coverage report found!" - fi + run: git tag ${{ steps.get-tag.outputs.bump }} - name: Build distribution - run: python -m build + run: uv build - name: Publish package distributions to PyPI if: steps.get-tag.outputs.bump != '' uses: pypa/gh-action-pypi-publish@release/v1 - name: Save git tag if: steps.get-tag.outputs.bump != '' run: git push origin ${{ steps.get-tag.outputs.bump }} - - name: Copy and surgery coverage - if: steps.get-tag.outputs.bump != '' - run: | - cp reports/.coverage . - sqlite3 .coverage "update file set path='src/' || substr(path, 40);" - name: Post coverage comment if: steps.get-tag.outputs.bump != '' uses: py-cov-action/python-coverage-comment-action@v3 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2a6a95978..7657eeb09 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -3,7 +3,6 @@ name: CI Workflow permissions: contents: write id-token: write - packages: read pull-requests: write concurrency: @@ -37,53 +36,37 @@ jobs: tox-${{ runner.os }}-3.12- test: - name: Build and test docker image + name: Test runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} steps: - name: Checkout repo uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - name: Configure docker - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin - - name: Get changed source files - id: changed-src - uses: tj-actions/changed-files@v41 + - name: Setup micromamba + uses: mamba-org/setup-micromamba@v1 with: - files: src/** - - name: Get changed base image files - if: steps.changed-src.outputs.any_changed == 'true' - id: changed-base - uses: tj-actions/changed-files@v41 + environment-file: environment.yml + create-args: python=3.12 + init-shell: bash + cache-downloads: true + cache-environment: true + post-cleanup: all + - name: Setup uv + uses: astral-sh/setup-uv@v6 with: - files: dockerfiles/base/* - - name: Pull base image and build - if: steps.changed-src.outputs.any_changed == 'true' && steps.changed-base.outputs.any_changed == 'true' - run: python flows/docker.py pull --build - - name: Pull base image - if: steps.changed-src.outputs.any_changed == 'true' && steps.changed-base.outputs.any_changed != 'true' - run: python flows/docker.py pull - - name: Build image - if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true' - run: python flows/docker.py build - - name: Run test image - if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true' - run: | - python flows/docker.py test - if ! test -f reports/.coverage; then - echo "Missing test coverage reports (reports/.coverage). Not all tests have passed?" - exit 1 - else - echo "Test coverage report found!" - fi - - name: Copy and surgery coverage - if: steps.changed-base.outputs.any_changed == 'true' || steps.changed-src.outputs.any_changed == 'true' - run: | - cp reports/.coverage . - sqlite3 .coverage "update file set path='src/' || substr(path, 40);" + enable-cache: true + - name: Install plinder + run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact + - name: Run tests + env: + PLINDER_OFFLINE: "true" + run: python -m pytest -n auto -v - name: Post coverage comment - if: steps.changed-src.outputs.any_changed == 'true' uses: py-cov-action/python-coverage-comment-action@v3 with: GITHUB_TOKEN: ${{ github.token }} @@ -106,7 +89,14 @@ jobs: cache-downloads: true cache-environment: true post-cleanup: all - - name: Install Python requirements - run: pip install six ".[docs,loader]" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - name: Install plinder + run: UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact --extra docs - name: Build the documentation run: sphinx-build --fail-on-warning docs build/docs + - name: Show notebook errors + if: failure() + run: cat build/docs/reports/examples/*.err.log 2>/dev/null || true diff --git a/.gitignore b/.gitignore index 5e6fc8223..313cf21ba 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,6 @@ tmp_foldseek tmp_mmseqs artifacts logs/ + +# uv lockfile: not committed; CI resolves the latest versions allowed by pyproject.toml +uv.lock diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..e4fba2183 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/README.md b/README.md index ed199b1b7..f0ecc9514 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,9 @@ More details about the competition and other helpful practical tips can be found ## πŸ”’ Plinder versions -We version the `plinder` dataset with two controls: - -- `PLINDER_RELEASE`: the month stamp of the last RCSB sync -- `PLINDER_RELEASE_NUMBER`: numbered release within that ingest month +We version the `plinder` dataset by the month stamp (`YYYY-MM`) of the RCSB +sync it was built from, set with `PLINDER_RELEASE`. The current release, +`2026-09`, is the default. We version the `plinder` application using an automated semantic versioning scheme based on the `git` commit history. @@ -57,7 +56,7 @@ with the dataset. #### Changelog: - WIP (Current β€” unreleased): - - **Public downloads on Cloudflare R2**: `2024-06/v2` now downloads from `https://plinderdata.org` instead of GCS; `PLINDER_MIRROR_URL` selects another HTTP mirror. Downloads stream to disk and verify size and MD5 before atomic replacement, interrupted transfers retry from the start, and cached files with the expected size are reused (force a refresh to repair same-size local corruption). Google storage dependencies are now optional: install `plinder[data]` for the GCS data-generation utilities. + - **Public downloads from Cameo**: releases download directly from `https://cameo3d.org/plinder/PLINDER-/` (currently `2026-09`); `PLINDER_MIRROR_URL` selects another HTTP file server with the same directory layout. Downloads stream to disk, validate HTTP Content-Length, and replace cached files atomically. Interrupted transfers retry from the start. Cached files are reused when their size and Last-Modified time match the server, so data hotfixes are fetched on the next access. Google storage dependencies are optional: install `plinder[data]` for the GCS data-generation utilities. - **Major backend refactor**: replaced OST, gemmi, plip, openbabel with biotite + peppr for data generation; removed 6 dependencies from ingest pipeline - **Nucleic acid support**: DNA/RNA chains now correctly included as receptor neighbors, mainchain/sidechain detection works for both protein and nucleic acids ([#61](https://github.com/plinder-org/plinder/issues/61)) - **Custom CIF support**: new `Entry.from_custom_cif_file` for structure-prediction outputs (Boltz, AlphaFold3, Chai-1) that ship CIFs without `_chem_comp_bond` ([#117](https://github.com/plinder-org/plinder/issues/117)). Bond orders come from `ligand_smiles_dict` via positional atom-order match (the convention these tools follow); element/count mismatches raise with the offending position, `force_substructure_match=True` opts into substructure matching when atom order isn't preserved. User SMILES win over CCD for both `smiles` and `resolved_stereo_matches_template` β€” closes a silent gap where biotite's `LIG` placeholder would pass any 3D conformer. Input CIFs are never mutated; optional `save_fixed_cif` persists the enriched copy. @@ -108,7 +107,7 @@ Moreover, as we enticipate this resource to be used for benchmarking a wide rang The *PLINDER* dataset is provided in two ways: - You can either use the files from the dataset directly using your preferred tooling - by downloading the data from `https://plinderdata.org` (Cloudflare R2), + by downloading the data from the [Cameo file server](https://cameo3d.org/plinder/PLINDER-2026-09/), - or you can utilize the dedicated `plinder` Python package for interfacing the data. @@ -118,14 +117,15 @@ After installing the package, download the index tables and cluster assignments for a release with: ```console -$ plinder_download --release 2024-06 --release-number v2 +$ plinder_download --release 2026-09 ``` The command offers the larger ligand, alignment, score, export, and search database groups separately, and dataset APIs also download required files -lazily. No Google Cloud credentials or SDK are needed: `2024-06/v2` downloads -from `https://plinderdata.org`; `PLINDER_MIRROR_URL` can select another HTTP -mirror. Other release versions are unavailable through the public downloader. +lazily. No Google Cloud credentials or SDK are needed: releases download +from `https://cameo3d.org/plinder/PLINDER-2026-09/`; `PLINDER_MIRROR_URL` +can select another HTTP file-server root containing `PLINDER-/` +directories. The legacy `2024-06/v2` layout requires an earlier `plinder` version. For details on the release paths, see [Documentation](https://plinder-org.github.io/plinder/tutorial/dataset.html). ## Installing the Python package diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 823953cba..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ -services: - - base: - image: ${IMAGE_REPO:-ghcr.io/plinder-org}/plinder-base:${BASE_TAG:-latest} - build: - context: ./dockerfiles/base - - plinder: - image: ${IMAGE_REPO:-ghcr.io/plinder-org}/plinder:${BUILD_TAG:-latest} - build: - context: . - dockerfile: ./dockerfiles/main/Dockerfile - args: - BASE_IMAGE: ${IMAGE_REPO:-ghcr.io/plinder-org}/plinder-base - BASE_TAG: ${BASE_TAG:-latest} - depends_on: - - base - - test: - image: ${IMAGE_REPO:-ghcr.io/plinder-org}/plinder:${BUILD_TAG:-latest} - depends_on: - - plinder - volumes: - - ./tests/:/app/tests/ - - ./docs/:/app/docs/ - - ./pytest.ini:/app/pytest.ini - - ./.coveragerc:/app/.coveragerc - - ./reports/:/app/reports/ - - ./examples/:/app/examples/ - command: /bin/bash -c "python -m pytest -n auto -v && cp .coverage reports/.coverage" diff --git a/dockerfiles/base/Dockerfile b/dockerfiles/base/Dockerfile deleted file mode 100644 index 1b43378bf..000000000 --- a/dockerfiles/base/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -FROM --platform=linux/amd64 mambaorg/micromamba:git-c160e88-jammy - -USER root -ENV DEBIAN_FRONTEND=noninteractive - -# Install OS dependencies -RUN apt-get --allow-releaseinfo-change update \ - && apt-get -y install --no-install-recommends \ - apt-utils \ - dialog 2>&1 \ - && apt-get install -y --no-install-recommends \ - git \ - gpg \ - wget \ - man-db \ - procps \ - tree \ - unzip \ - gcc \ - build-essential \ - lsb-release \ - curl \ - vim \ - exuberant-ctags \ - apt-transport-https \ - ca-certificates \ - gnupg \ - sudo \ - libgl1-mesa-glx \ - libxrender1 \ - rsync \ - libtiff-dev \ - && apt-get autoclean \ - && apt-get autoremove \ - && rm -rf /var/lib/apt/lists/* - -# Install rust toolchain -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -ENV PATH=/root/.cargo/bin:${PATH} - -# Install conda env -COPY env.yml /tmp/env.yml -RUN micromamba install -y -f /tmp/env.yml && \ - micromamba clean --all --yes && \ - rm /tmp/env.yml - -# Install gsutil without full gcloud sdk -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | \ - tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \ - && apt-get update -y \ - && apt-get install google-cloud-cli -y \ - && apt-get autoclean \ - && apt-get autoremove \ - && rm -rf /var/lib/apt/lists/* - -ENV BASH_ENV=/usr/local/bin/_activate_current_env.sh -ENV LD_LIBRARY_PATH=/opt/conda/lib - -COPY requirements.txt /tmp/requirements.txt -RUN --mount=type=secret,id=INDEX_URL \ - PIP_KEYRING_PROVIDER=subprocess python -m pip install \ - --extra-index-url "$(cat /run/secrets/INDEX_URL)" \ - --no-cache-dir -r /tmp/requirements.txt && \ - rm /tmp/requirements.txt - -ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] - -CMD ["/bin/bash"] diff --git a/dockerfiles/base/env.yml b/dockerfiles/base/env.yml deleted file mode 100644 index f3d4541f2..000000000 --- a/dockerfiles/base/env.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: base -channels: - - conda-forge - - defaults - - bioconda -dependencies: - - python=3.12.* - - pyopenssl=23.2.0 - - requests=2.25.1 - - google-cloud-storage - - gcsfs - - reduce - - openstructure>=2.12.0 - - mmseqs2 - - foldseek - - pip: - - keyrings.google-artifactregistry-auth==1.1.2 diff --git a/dockerfiles/base/requirements.txt b/dockerfiles/base/requirements.txt deleted file mode 100644 index 55714adac..000000000 --- a/dockerfiles/base/requirements.txt +++ /dev/null @@ -1,185 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --annotation-style=line --extra=test,pipeline,plots --output-file=/output/requirements.txt --strip-extras pyproject.toml -# -aiohappyeyeballs==2.4.0 # via aiohttp -aiohttp==3.10.5 # via gcsfs -aiosignal==1.3.1 # via aiohttp -annotated-types==0.7.0 # via pydantic -antlr4-python3-runtime==4.9.3 # via omegaconf -anyio==4.4.0 # via httpx, jupyter-server -argon2-cffi==23.1.0 # via jupyter-server -argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.3.0 # via isoduration -asttokens==2.4.1 # via stack-data -async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 # via aiohttp -attrs==24.2.0 # via aiohttp, jsonschema, pytest, referencing -babel==2.16.0 # via jupyterlab-server -beautifulsoup4==4.12.3 # via nbconvert -biotite==1.0.1 # via plinder (pyproject.toml) -bleach==6.1.0 # via nbconvert -build==0.9.0 # via plinder (pyproject.toml) -cachetools==5.5.0 # via google-auth, tox -certifi==2024.8.30 # via httpcore, httpx, requests -cffi==1.17.0 # via argon2-cffi-bindings -chardet==5.2.0 # via tox -charset-normalizer==3.3.2 # via requests -cloudpathlib==0.19.0 # via plinder (pyproject.toml) -colorama==0.4.6 # via tox -comm==0.2.2 # via ipykernel, ipywidgets -contourpy==1.3.0 # via matplotlib -coverage==7.6.1 # via pytest-cov -cycler==0.12.1 # via matplotlib -debugpy==1.8.5 # via ipykernel -decorator==5.1.1 # via gcsfs, ipython -defusedxml==0.7.1 # via nbconvert -distlib==0.3.8 # via virtualenv -duckdb==1.5.2 # via plinder (pyproject.toml) -eval-type-backport==0.2.0 # via plinder (pyproject.toml) -exceptiongroup==1.2.2 # via anyio, ipython, pytest -execnet==2.1.1 # via pytest-xdist -executing==2.1.0 # via stack-data -fastjsonschema==2.20.0 # via nbformat -filelock==3.15.4 # via tox, virtualenv -fonttools==4.53.1 # via matplotlib -fqdn==1.5.1 # via jsonschema -frozenlist==1.4.1 # via aiohttp, aiosignal -fsspec==2024.6.1 # via gcsfs -future==1.0.0 # via mmcif -gcsfs==2024.6.1 # via plinder (pyproject.toml) -gemmi==0.6.6 # via plinder (pyproject.toml) -google-api-core==2.19.2 # via google-cloud-core, google-cloud-storage -google-auth==2.34.0 # via gcsfs, google-api-core, google-auth-oauthlib, google-cloud-core, google-cloud-storage -google-auth-oauthlib==1.2.1 # via gcsfs -google-cloud-core==2.4.1 # via google-cloud-storage -google-cloud-storage==2.18.2 # via gcsfs, plinder (pyproject.toml) -google-crc32c==1.6.0 # via google-cloud-storage, google-resumable-media -google-resumable-media==2.7.2 # via google-cloud-storage -googleapis-common-protos==1.65.0 # via google-api-core -h11==0.14.0 # via httpcore -httpcore==1.0.5 # via httpx -httpx==0.27.2 # via jupyterlab -idna==3.8 # via anyio, httpx, jsonschema, requests, yarl -importlib-metadata==8.4.0 # via jupyter-client, jupyter-lsp, jupyterlab, jupyterlab-server, nbconvert -importlib-resources==6.4.4 # via matplotlib -iniconfig==2.0.0 # via pytest -ipykernel==6.29.5 # via jupyterlab -ipython==8.18.1 # via ipykernel, ipywidgets -ipython-genutils==0.2.0 # via ipywidgets -ipywidgets==7.8.4 # via mols2grid -isoduration==20.11.0 # via jsonschema -jedi==0.19.1 # via ipython -jinja2==3.1.4 # via jupyter-server, jupyterlab, jupyterlab-server, mols2grid, nbconvert -json5==0.9.25 # via jupyterlab-server -jsonpointer==3.0.0 # via jsonschema -jsonschema==4.23.0 # via jupyter-events, jupyterlab-server, nbformat -jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.2 # via ipykernel, jupyter-server, nbclient -jupyter-core==5.7.2 # via ipykernel, jupyter-client, jupyter-server, jupyterlab, nbclient, nbconvert, nbformat -jupyter-events==0.10.0 # via jupyter-server -jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.2 # via jupyter-lsp, jupyterlab, jupyterlab-server, notebook, notebook-shim -jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.5 # via notebook -jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.3 # via jupyterlab, notebook -jupyterlab-widgets==1.1.10 # via ipywidgets -kiwisolver==1.4.5 # via matplotlib -markupsafe==2.1.5 # via jinja2, nbconvert -matplotlib==3.9.2 # via matplotlib-venn, plinder (pyproject.toml), seaborn -matplotlib-inline==0.1.7 # via ipykernel, ipython -matplotlib-venn==1.1.1 # via plinder (pyproject.toml) -mistune==3.0.2 # via nbconvert -mmcif==0.89.0 # via plinder (pyproject.toml) -mols2grid==2.0.0 # via plinder (pyproject.toml) -msgpack==1.0.8 # via biotite, mmcif -multidict==6.0.5 # via aiohttp, yarl -nbclient==0.10.0 # via nbconvert -nbconvert==7.16.4 # via jupyter-server -nbformat==5.10.4 # via jupyter-server, nbclient, nbconvert, plinder (pyproject.toml) -nest-asyncio==1.6.0 # via ipykernel -networkx==3.2.1 # via biotite -notebook==7.2.2 # via widgetsnbextension -notebook-shim==0.2.4 # via jupyterlab, notebook -numpy==1.26.4 # via biotite, contourpy, matplotlib, matplotlib-venn, mols2grid, pandas, plinder (pyproject.toml), posebusters, pyarrow, rdkit, scipy, seaborn -oauthlib==3.2.2 # via requests-oauthlib -omegaconf==2.3.0 # via plinder (pyproject.toml) -overrides==7.7.0 # via jupyter-server -packaging==24.1 # via build, ipykernel, jupyter-server, jupyterlab, jupyterlab-server, matplotlib, nbconvert, plotly, pyproject-api, pytest, setuptools-scm, tox -pandas==2.2.2 # via mols2grid, plinder (pyproject.toml), posebusters, seaborn -pandocfilters==1.5.1 # via nbconvert -parso==0.8.4 # via jedi -pep517==0.13.1 # via build -pexpect==4.9.0 # via ipython -pillow==10.4.0 # via matplotlib, rdkit -platformdirs==4.2.2 # via jupyter-core, tox, virtualenv -plotly==5.24.0 # via plinder (pyproject.toml) -pluggy==1.5.0 # via pytest, tox -posebusters==0.3.1 # via plinder (pyproject.toml) -prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.47 # via ipython -proto-plus==1.24.0 # via google-api-core -protobuf==5.28.0 # via google-api-core, googleapis-common-protos, proto-plus -psutil==6.0.0 # via ipykernel -ptyprocess==0.7.0 # via pexpect, terminado -pure-eval==0.2.3 # via stack-data -pyarrow==17.0.0 # via plinder (pyproject.toml) -pyasn1==0.6.0 # via pyasn1-modules, rsa -pyasn1-modules==0.4.0 # via google-auth -pycparser==2.22 # via cffi -pydantic==2.8.2 # via plinder (pyproject.toml) -pydantic-core==2.20.1 # via pydantic -pygments==2.18.0 # via ipython, nbconvert -pyparsing==3.1.4 # via matplotlib -pyproject-api==1.7.1 # via tox -pytest==7.2.0 # via plinder (pyproject.toml), pytest-cov, pytest-xdist -pytest-cov==4.0.0 # via plinder (pyproject.toml) -pytest-xdist==3.6.1 # via plinder (pyproject.toml) -python-dateutil==2.9.0.post0 # via arrow, jupyter-client, matplotlib, pandas -python-json-logger==2.0.7 # via jupyter-events -pytz==2024.1 # via pandas -pyyaml==6.0.2 # via jupyter-events, omegaconf, posebusters -pyzmq==26.2.0 # via ipykernel, jupyter-client, jupyter-server -rdkit>=2023.9.5 # via plinder (pyproject.toml), posebusters -referencing==0.35.1 # via jsonschema, jsonschema-specifications, jupyter-events -requests==2.32.3 # via biotite, gcsfs, google-api-core, google-cloud-storage, jupyterlab-server, mmcif, requests-oauthlib -requests-oauthlib==2.0.0 # via google-auth-oauthlib -rfc3339-validator==0.1.4 # via jsonschema, jupyter-events -rfc3986-validator==0.1.1 # via jsonschema, jupyter-events -rpds-py==0.20.0 # via jsonschema, referencing -rsa==4.9 # via google-auth -scipy==1.13.1 # via matplotlib-venn, plinder (pyproject.toml) -seaborn==0.13.2 # via plinder (pyproject.toml) -send2trash==1.8.3 # via jupyter-server -setuptools-scm==7.0.5 # via plinder (pyproject.toml) -six==1.16.0 # via asttokens, bleach, mmcif, plinder (pyproject.toml), python-dateutil, rfc3339-validator -sniffio==1.3.1 # via anyio, httpx -soupsieve==2.6 # via beautifulsoup4 -stack-data==0.6.3 # via ipython -tenacity==9.0.0 # via plotly -terminado==0.18.1 # via jupyter-server, jupyter-server-terminals -tinycss2==1.3.0 # via nbconvert -tomli==2.0.1 # via build, coverage, jupyterlab, pep517, pyproject-api, pytest, setuptools-scm, tox -tornado==6.4.1 # via ipykernel, jupyter-client, jupyter-server, jupyterlab, notebook, terminado -tox==4.18.0 # via plinder (pyproject.toml) -tqdm==4.66.5 # via plinder (pyproject.toml) -traitlets==5.14.3 # via comm, ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, jupyter-events, jupyter-server, jupyterlab, matplotlib-inline, nbclient, nbconvert, nbformat -types-python-dateutil==2.9.0.20240821 # via arrow -typing-extensions==4.12.2 # via anyio, async-lru, cloudpathlib, ipython, plinder (pyproject.toml), pydantic, pydantic-core, setuptools-scm -tzdata==2024.1 # via pandas -uri-template==1.3.0 # via jsonschema -urllib3==2.2.2 # via requests -virtualenv==20.26.3 # via tox -wcwidth==0.2.13 # via prompt-toolkit -webcolors==24.8.0 # via jsonschema -webencodings==0.5.1 # via bleach, tinycss2 -websocket-client==1.8.0 # via jupyter-server -widgetsnbextension==3.6.9 # via ipywidgets -yarl==1.9.7 # via aiohttp -zipp==3.20.1 # via importlib-metadata, importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/dockerfiles/main/Dockerfile b/dockerfiles/main/Dockerfile deleted file mode 100644 index 1b449c6c3..000000000 --- a/dockerfiles/main/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG BASE_IMAGE -ARG BASE_TAG -FROM --platform=linux/amd64 ${BASE_IMAGE}:${BASE_TAG} -ARG MAMBA_DOCKERFILE_ACTIVATE=1 -ARG DEPENDENCY_BLOCKS=data,test,eval - -WORKDIR /app -COPY requirements_data.txt requirements_data.txt -RUN python -m pip install -r requirements_data.txt -COPY src src -COPY pyproject.toml pyproject.toml -RUN --mount=source=.git,target=.git,type=bind \ - --mount=type=secret,id=INDEX_URL \ - PIP_KEYRING_PROVIDER=subprocess python -m pip install \ - --extra-index-url "$(cat /run/secrets/INDEX_URL)" \ - --no-cache-dir '.['"${DEPENDENCY_BLOCKS}"']' diff --git a/dockerfiles/test/Dockerfile b/dockerfiles/test/Dockerfile deleted file mode 100644 index 454518404..000000000 --- a/dockerfiles/test/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -ARG BASE_IMAGE -ARG BASE_TAG -FROM --platform=linux/amd64 ${BASE_IMAGE}:${BASE_TAG} -ARG MAMBA_DOCKERFILE_ACTIVATE=1 -COPY src src -COPY pyproject.toml pyproject.toml -RUN --mount=source=.git,target=.git,type=bind \ - python -m pip install --no-cache-dir . diff --git a/docs/conf.py b/docs/conf.py index 84be4b128..c425beb25 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,23 +64,6 @@ nb_custom_formats = {".ipynb": ["jupytext.reads", {"fmt": "ipynb"}]} nb_execution_timeout = 720 nb_kernel_rgx_aliases = {"plinder.*": "python3"} -# TODO(docs-v3): these notebooks were migrated to the v3 schema (new cluster -# column names, system_receptor_type, structure APIs) but the docs build -# executes them against the published v2 dataset, so re-execution fails. Skip -# executing them (committed outputs are rendered instead) until the v3 dataset -# is published; then remove this list to restore live execution. -nb_execution_excludepatterns = [ - "examples/2_query_filter_index.ipynb", - "examples/3_access_system_files.ipynb", - "examples/4_align_mask_crop.ipynb", - "examples/5_dataset_and_loader.ipynb", - # TODO: v3-only: these fetch 2026-07 release artifacts (systems, interfaces, - # similarity exports) that are not published yet. - "examples/custom_scoring.ipynb", - "examples/evaluation.ipynb", - "examples/protein_interfaces.ipynb", - "examples/similarity_and_representatives.ipynb", -] myst_enable_extensions = [ "amsmath", "attrs_inline", @@ -166,6 +149,7 @@ } html_sidebars = { # No primary sidebar for these pages + "tutorial/**": [], "dataset": [], "evaluation": [], "citation": [], diff --git a/docs/contribution/development.md b/docs/contribution/development.md index 01a08a39e..826400c0c 100644 --- a/docs/contribution/development.md +++ b/docs/contribution/development.md @@ -16,8 +16,8 @@ $ git clone https://github.com/plinder-org/plinder.git ### Creating the Conda environment -The data generation pipeline (`plinder.data`) requires a few tools that are only -available via _Conda_ (mmseqs2, foldseek, reduce). +The data generation pipeline (`plinder.data`) and evaluation require a few tools +that are only available via _Conda_ (mmseqs2, foldseek, OpenStructure). If you have not _Conda_ installed yet, we recommend its installation via [miniforge](https://github.com/conda-forge/miniforge). @@ -28,18 +28,19 @@ $ mamba activate plinder ### Installing `plinder` -With [uv](https://docs.astral.sh/uv/), a single command installs `plinder` in -editable mode with the `dev`, `eval` and `loader` extras and the data pipeline -dependencies from `requirements_data.txt`: +All Python dependencies are declared in `pyproject.toml`. +With [uv](https://docs.astral.sh/uv/), install `plinder` in editable mode with the +complete `dev` dependency group (all extras used in CI, including CPU-only pytorch +on Linux and the git-only pipeline packages) into the active Conda environment: ```console -$ uv sync +$ UV_PROJECT_ENVIRONMENT="$CONDA_PREFIX" uv sync --inexact ``` -No separate `pip install -r requirements_data.txt` step is needed. -The Conda-only tools above are still required. +CI runs the same command. Without the Conda environment, a plain `uv sync` +installs into `.venv`; tests that need the Conda-only tools will then fail. -With pip, the base install covers data generation and the core library (numpy 2 compatible): +With pip, the base install covers data generation and the core library: ```console $ pip install -e ".[dev]" @@ -62,16 +63,8 @@ is installed by `mamba env create -f environment.yml` above. Data generation (`plinder.data`) does **not** require OpenStructure and works with numpy 2. -For the full data pipeline, pip installs need additional dependencies -(`uv sync` already includes them): - -```console -$ pip install -r requirements_data.txt -``` - -This includes Linux pytorch (for the loader) and pipeline-specific tools. -For Windows and MacOS users, please see the relevant -[_Docker_](#docker-target) resources. +The full data pipeline also needs the git-only packages in the `dev` +dependency group, which only `uv sync` installs. ::: ### Enabling Pre-commit hooks @@ -82,20 +75,6 @@ Please install pre-commit hooks, that will run the same code quality checks as t $ pre-commit install ``` -(docker-target)= -### Alternative: Using a Docker container - -We also publish the `plinder` project as a -[docker container](https://github.com/plinder-org/plinder/pkgs/container/plinder) -as alternative to the _Conda_-based installation, to ensure the highest level of -compatibility with non-Linux platforms. -See the relevant docker resources here for more details: - -- `docker-compose.yml`: defines a `base` image, the `plinder` "app" and a `test` - container -- `dockerfiles/base/`: contains the files for the `base` image -- `dockerfiles/main/`: contains the files for the `plinder` "app" image - ## Testing and linting `plinder` uses [`tox`](https://tox.wiki) for running tests, type checks and code linting diff --git a/docs/contribution/release.md b/docs/contribution/release.md index b059375cf..37065b21e 100644 --- a/docs/contribution/release.md +++ b/docs/contribution/release.md @@ -19,6 +19,4 @@ The CI workflow will use the **most recent** match in the commit history to make Each new version release automatically triggers the following platforms: - A new *PyPI* release is created. -- A new *Docker* image is pushed to the - [registry](https://github.com/plinder-org/plinder/pkgs/container/plinder). - This documentation website is updated. diff --git a/docs/dataset.md b/docs/dataset.md index 52ed04303..d38441e44 100644 --- a/docs/dataset.md +++ b/docs/dataset.md @@ -9,8 +9,9 @@ extracts protein-ligand systems and protein-protein interfaces from the PDB. ## Release layout -PLINDER releases are identified by the month of their PDB snapshot and a -release number within that month. The file layout is: +PLINDER releases are identified by the month of their PDB snapshot +(`YYYY-MM`; the current release is `2026-09`) and served from +`https://cameo3d.org/plinder/PLINDER-/`. The file layout is: :::{note} Historical releases remain available at `gs://plinder/2024-04/v0`, @@ -19,54 +20,60 @@ differ from the current release. ::: ```text -/ -└── / - β”œβ”€β”€ index/ - β”‚ β”œβ”€β”€ annotation_table.parquet - β”‚ β”œβ”€β”€ system_validation.parquet - β”‚ β”œβ”€β”€ entry_chains.parquet - β”‚ β”œβ”€β”€ entry_biounit_chains.parquet - β”‚ β”œβ”€β”€ entry_metadata.parquet - β”‚ β”œβ”€β”€ entry_sources.parquet - β”‚ β”œβ”€β”€ interface_annotation_table.parquet - β”‚ β”œβ”€β”€ alignment_chain_lookup.parquet - β”‚ β”œβ”€β”€ linked_apo_structures.parquet - β”‚ β”œβ”€β”€ interface_apo_structures.parquet - β”‚ β”œβ”€β”€ ligand_pocket_membership.parquet - β”‚ β”œβ”€β”€ ligand_pocket_representatives.parquet - β”‚ β”œβ”€β”€ ligand_clusters.parquet - β”‚ β”œβ”€β”€ ligand_affinity.parquet - β”‚ β”œβ”€β”€ bindingdb_measurements.parquet - β”‚ β”œβ”€β”€ ligand_mmp_pairs.parquet - β”‚ β”œβ”€β”€ interface_half_representatives.parquet - β”‚ β”œβ”€β”€ interface_membership.parquet - β”‚ β”œβ”€β”€ interface_representatives.parquet - β”‚ └── interface_clusters.parquet - β”œβ”€β”€ ligand_archives/ - β”‚ β”œβ”€β”€ {two_char_code}.parquet - β”‚ └── manifest.json - β”œβ”€β”€ alignment_cigars/ - β”‚ └── search_db=holo/ - β”‚ └── alignment_type={foldseek,mmseqs}/ - β”‚ └── shard={two_char_code}.parquet - β”œβ”€β”€ ligand_scores/ - β”‚ └── {fragment}.parquet - β”œβ”€β”€ ligand_sampling/ - β”‚ β”œβ”€β”€ set_cover/metric={metric}/threshold={threshold}.parquet - β”‚ └── directed_set_cover/metric={metric}/threshold={threshold}.parquet - β”œβ”€β”€ interface_sampling/ - β”‚ └── directed_set_cover/metric=interface_qcov/threshold={threshold}.parquet - β”œβ”€β”€ search_databases/ - β”‚ β”œβ”€β”€ manifest.json - β”‚ β”œβ”€β”€ holo_foldseek/ - β”‚ └── holo_mmseqs/ - └── exports/ - β”œβ”€β”€ ligand_similarity_scores.parquet - β”œβ”€β”€ interface_similarity_scores.parquet - β”œβ”€β”€ interface_half_similarity_scores.parquet - └── protein_similarity_scores/ - └── alignment_type={foldseek,mmseqs}/ - └── shard={two_char_code}.parquet +PLINDER-/ +β”œβ”€β”€ index/ +β”‚ β”œβ”€β”€ annotation_table.parquet +β”‚ β”œβ”€β”€ system_validation.parquet +β”‚ β”œβ”€β”€ entry_chains.parquet +β”‚ β”œβ”€β”€ entry_biounit_chains.parquet +β”‚ β”œβ”€β”€ entry_metadata.parquet +β”‚ β”œβ”€β”€ entry_sources.parquet +β”‚ β”œβ”€β”€ interface_annotation_table.parquet +β”‚ β”œβ”€β”€ alignment_chain_lookup.parquet +β”‚ β”œβ”€β”€ linked_apo_structures.parquet +β”‚ β”œβ”€β”€ interface_apo_structures.parquet +β”‚ β”œβ”€β”€ ligand_pocket_membership.parquet +β”‚ β”œβ”€β”€ ligand_pocket_residues.parquet +β”‚ β”œβ”€β”€ ligand_pocket_representatives.parquet +β”‚ β”œβ”€β”€ ligand_clusters.parquet +β”‚ β”œβ”€β”€ ligand_affinity.parquet +β”‚ β”œβ”€β”€ bindingdb_measurements.parquet +β”‚ β”œβ”€β”€ ligand_mmp_pairs.parquet +β”‚ β”œβ”€β”€ interface_half_representatives.parquet +β”‚ β”œβ”€β”€ interface_membership.parquet +β”‚ β”œβ”€β”€ interface_representatives.parquet +β”‚ └── interface_clusters.parquet +β”œβ”€β”€ protein_clusters/ +β”‚ β”œβ”€β”€ sequence.parquet +β”‚ └── structure.parquet +β”œβ”€β”€ ligand_archives/ +β”‚ └── {two_char_code}.parquet +β”œβ”€β”€ alignment_cigars/ +β”‚ └── search_db={holo,apo,interface_apo}/ +β”‚ └── alignment_type={foldseek,mmseqs}/ +β”‚ └── shard={two_char_code}.parquet +β”œβ”€β”€ ligand_scores/ +β”‚ └── retained.parquet +β”œβ”€β”€ ligand_sampling/ +β”‚ β”œβ”€β”€ set_cover/metric={metric}/threshold={threshold}.parquet +β”‚ └── directed_set_cover/metric={metric}/threshold={threshold}.parquet +β”œβ”€β”€ interface_sampling/ +β”‚ └── directed_set_cover/metric={interface_qcov,interface_side_qcov}/threshold={threshold}.parquet +β”œβ”€β”€ search_databases/ +β”‚ β”œβ”€β”€ holo_foldseek/ +β”‚ β”œβ”€β”€ holo_mmseqs/ +β”‚ β”œβ”€β”€ monomer_foldseek/ +β”‚ └── monomer_mmseqs/ +└── exports/ + β”œβ”€β”€ ligand_similarity_scores.parquet + β”œβ”€β”€ interface_similarity_scores.parquet + β”œβ”€β”€ interface_half_similarity_scores.parquet + β”œβ”€β”€ protein_similarity_scores/ + β”‚ └── alignment_type={foldseek,mmseqs}/ + β”‚ └── shard={two_char_code}.parquet + └── monomer_similarity_scores/ + └── alignment_type={foldseek,mmseqs}/ + └── monomer_{holo,monomer}_{batch}.parquet ``` `plinder_download` downloads the index and representative-cover tables by @@ -234,6 +241,15 @@ exact assembly-chain instance selected for reconstruction. chain-pair scores, including query and target coverage, sequence identity, sequence similarity, and Foldseek lDDT. Percentage values are stored as integers from 0 to 100. +`exports/monomer_similarity_scores/` adds query chains outside ligand pockets +and protein interfaces. Pass `include_monomers=True` to +`query_protein_similarity()` to read both sets of chain pairs. + +For a custom structure or FASTA search, `search()` returns +`result.chain_similarity_scores`, a Parquet file of direct chain-level hits +with the same score fields. FASTA results retain the original sequence IDs. +By default, chain hits also include protein chains outside ligand pockets +and protein interfaces. Pocket and interface scores use their own targets. The optional `alignment_cigars/` files contain full-chain CIGARs, alignment starts, and bit-packed residue identity flags. Holo chain-pair statistics are @@ -279,12 +295,13 @@ in `::side=1` or `::side=2`. Use `query_interface_similarity()` or ## Representative covers -- `ligand_sampling/set_cover/` contains an undirected set cover for reciprocal - Tanimoto similarity; +- `ligand_sampling/set_cover/` contains undirected set covers for reciprocal + Tanimoto (ECFP4) and Jaccard (MHFP6) similarity; - `ligand_sampling/directed_set_cover/` contains directed covers for pocket, interaction, and pocket-weighted ligand 3D metrics; -- `interface_sampling/directed_set_cover/` contains directed covers for protein - interfaces. +- `interface_sampling/directed_set_cover/` contains directed covers for whole + protein interfaces (`interface_qcov`) and interface sides + (`interface_side_qcov`). Each metric has files named `metric={metric}/threshold={threshold}.parquet`. The release also includes one row per ligand in diff --git a/docs/examples/1_download.ipynb b/docs/examples/1_download.ipynb index a116b0491..6c6283b33 100644 --- a/docs/examples/1_download.ipynb +++ b/docs/examples/1_download.ipynb @@ -4,49 +4,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Downloading the dataset\n", + "# Download the full dataset\n", "\n", - "The default location for the dataset is `~/.local/share/plinder//` where:\n", - "- `` is the month of the PDB snapshot, formatted as `YYYY-MM`;\n", - "- `` distinguishes PLINDER releases from the same month.\n", - "\n", - "Set the `PLINDER_MOUNT` environment variable to use a different location." + "Use `plinder_download` when you need a local copy, for example on a compute node without internet access." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "## Choose a local directory\n", "\n", - "## Use `plinder_download` to download the complete dataset\n", - "\n", - "```bash\n", - "plinder_download --help\n", - "usage:\n", - " Download the full plinder dataset for the current configuration.\n", - " Note that even though this is wrapped in a progress bar, the estimated\n", - " completion time can vary wildly as it iterates over larger files vs.\n", - " smaller ones.\n", - " \n", - "\n", - "optional arguments:\n", - " -h, --help show this help message and exit\n", - " --release RELEASE plinder release\n", - " --release-number RELEASE_NUMBER\n", - " plinder release number\n", - " -y, --yes skip confirmation\n", - "```\n", - "\n", - ":::{note} The public `2024-06/v2` release is served from Cloudflare R2 and contains about 220 GB of downloadable files. Allow additional space for extracted archives; download time depends on your connection. Completed files are reused on subsequent runs. Set `PLINDER_OFFLINE=true` to use the cache without network requests. :::" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Use the `plinder` Python package to lazily access the dataset\n", - "\n", - "Alternatively, if the `PLINDER_OFFLINE` environment variable **unset** (which is the default empty), the dataset will be downloaded on lazily and on the fly as you access the data. This is preferred for exploration and prototyping as you don't need to download the entire dataset at once and can just work with the assets you need for your use-case." + "Files are cached in `~/.local/share/plinder/` by default. Set `PLINDER_MOUNT` before importing PLINDER to choose another location. Inspect the configured directory with:" ] }, { @@ -55,20 +24,19 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", + "from plinder.core import get_config\n", "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" + "cfg = get_config()\n", + "print(cfg.data.plinder_dir)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Use `plinder_download` to select release files\n", + "## Download release files\n", "\n", - "The command describes the available release and confirmation options:\n", - "" + "`plinder_download` downloads the index and representative-cover tables, then asks which larger groups to include. The [dataset reference](../dataset.md) lists these groups." ] }, { @@ -84,29 +52,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The [dataset reference](../dataset.md) lists the default and optional download groups. After preparing the required files, set `PLINDER_OFFLINE=true` to prevent network access." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Access release files from Python\n", + "Run this in a terminal to include every group without confirmation:\n", "\n", - "Alternatively, when `PLINDER_OFFLINE` is unset, API calls fetch missing release files as they are requested. This is useful for exploration because only the files needed by the current query or reconstruction are cached." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from plinder.core import get_config\n", + "```bash\n", + "plinder_download --yes\n", + "```\n", "\n", - "cfg = get_config()\n", - "print(f\"local directory: {cfg.data.plinder_dir}\")\n", - "print(f\"remote data directory: {cfg.data.plinder_remote}\")" + "Rerun the same command after an interruption; completed files are reused. Source PDB mmCIFs are fetched separately when reconstructing structures. For offline reconstruction, first cache the entries you need with `download_pdb_mmcifs()`, described in the {ref}`structure reference `.\n", + "\n", + "## Work offline\n", + "\n", + "After the files you need are available locally, set this before starting Python:\n", + "\n", + "```bash\n", + "export PLINDER_OFFLINE=true\n", + "```" ] } ], @@ -117,18 +77,10 @@ "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/docs/examples/2_query_filter_index.ipynb b/docs/examples/2_query_filter_index.ipynb index d13bc6747..4316f9ff2 100644 --- a/docs/examples/2_query_filter_index.ipynb +++ b/docs/examples/2_query_filter_index.ipynb @@ -10,19 +10,6 @@ "Use {func}`plinder.core.query_table` to select and filter release records. The [dataset reference](../dataset.md) defines every table, key, relationship, and column." ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "select-release", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/docs/examples/3_access_system_files.ipynb b/docs/examples/3_access_system_files.ipynb index 736072fbc..b9237ec36 100644 --- a/docs/examples/3_access_system_files.ipynb +++ b/docs/examples/3_access_system_files.ipynb @@ -7,18 +7,6 @@ "# Access system files with PlinderSystem" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" - ] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/examples/4_align_mask_crop.ipynb b/docs/examples/4_align_mask_crop.ipynb index f876d3044..a6cee989b 100644 --- a/docs/examples/4_align_mask_crop.ipynb +++ b/docs/examples/4_align_mask_crop.ipynb @@ -9,18 +9,6 @@ "This advanced guide starts from a reconstructed ligand system and shows the in-memory coordinate views used for featurization and structural comparison. For file access and linked apo reconstruction, begin with [Access system files with PlinderSystem](3_access_system_files.ipynb)." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/docs/examples/custom_scoring.ipynb b/docs/examples/custom_scoring.ipynb index c235cdb0b..ffde8f4b0 100644 --- a/docs/examples/custom_scoring.ipynb +++ b/docs/examples/custom_scoring.ipynb @@ -31,9 +31,6 @@ "import os\n", "from pathlib import Path\n", "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")\n", - "\n", "import pandas as pd\n", "\n", "from plinder.core import PlinderSystem, PlinderRelease\n", diff --git a/docs/examples/evaluation.ipynb b/docs/examples/evaluation.ipynb index 8a5ac39ba..1785a3c92 100644 --- a/docs/examples/evaluation.ipynb +++ b/docs/examples/evaluation.ipynb @@ -44,9 +44,6 @@ "import yaml\n", "import pandas as pd\n", "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")\n", - "\n", "from plinder.core import PlinderRelease\n", "from plinder.eval import evaluate_predictions\n", "\n", diff --git a/docs/examples/index.md b/docs/examples/index.md index 561eb8730..9555b3e2b 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -10,7 +10,6 @@ Start here when working directly with the release tables and files. ```{toctree} :maxdepth: 1 -1_download 2_query_filter_index ``` @@ -48,3 +47,13 @@ Compare predicted ligand poses and protein interfaces with known references. evaluation ``` + +## Offline access + +Download a local copy when you need to work without internet access. + +```{toctree} +:maxdepth: 1 + +1_download +``` diff --git a/docs/examples/protein_interfaces.ipynb b/docs/examples/protein_interfaces.ipynb index 0328f8f1c..efaf6c37e 100644 --- a/docs/examples/protein_interfaces.ipynb +++ b/docs/examples/protein_interfaces.ipynb @@ -9,18 +9,6 @@ "Start from a selected interface ID and rebuild its two-chain coordinate view. Use [Query the release tables](2_query_filter_index.ipynb) to select interfaces and {ref}`Protein interfaces ` for the column definitions." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/docs/examples/similarity_and_representatives.ipynb b/docs/examples/similarity_and_representatives.ipynb index 56b0cd09e..b52decf4c 100644 --- a/docs/examples/similarity_and_representatives.ipynb +++ b/docs/examples/similarity_and_representatives.ipynb @@ -6,19 +6,7 @@ "source": [ "# Similarity scores and representative covers\n", "\n", - "Reconstruct selected pairwise scores, read part of a complete score table, and query representative assignments. {ref}`Similarity scores ` defines the metrics and files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ.setdefault(\"PLINDER_RELEASE\", \"2026-07\")\n", - "os.environ.setdefault(\"PLINDER_RELEASE_NUMBER\", \"1\")" + "Compare ligands and protein interfaces, explore similarity scores, and find representative clusters. {ref}`Similarity scores ` defines the metrics and files." ] }, { @@ -53,7 +41,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Reconstruct ligand similarities\n", + "## Compare ligand similarities\n", "\n", "`reconstruct_similarity_scores()` reads the files needed for the requested query and target IDs. This example uses a one-pair self-comparison to keep execution small." ] @@ -110,7 +98,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Reconstruct protein-interface similarities\n", + "## Compare protein-interface similarities\n", "\n", "The interface API likewise accepts selected query and target IDs." ] diff --git a/docs/index.md b/docs/index.md index 22d1d1950..ee8714b43 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,7 +11,7 @@ html_theme.sidebar_secondary.remove: true Resource**: a comprehensive, annotated, high-quality resource for training and evaluating protein-ligand and protein-protein structure models. -- \> 400k PLI systems across > 11k SCOP domains and > 50k unique small molecules +- 611,930 protein–ligand systems, 803,190 proper ligands, and 3,811,623 protein–protein interfaces - Ligand and protein-interface annotations plus compact entry, chain, and representative tables - Automated curation pipeline to keep up with the PDB - Reusable ligand, pocket, and protein-interface similarities and representative assignments @@ -37,7 +37,7 @@ Install PLINDER, select a release, and continue to the runnable guides. -Download release files and query columns across the ligand, interface, +Query columns across the ligand, interface, entry, chain, and representative tables. ::::: @@ -86,6 +86,5 @@ dataset evaluation api/index contribution/index -migration/r2 citation ::: diff --git a/docs/migration/r2.md b/docs/migration/r2.md deleted file mode 100644 index bcfad88de..000000000 --- a/docs/migration/r2.md +++ /dev/null @@ -1,84 +0,0 @@ -# Static release downloads from R2 - -Public dataset reads for `2024-06/v2` use `https://plinderdata.org`. No S3 keys -or Google credentials are required. Other online release/bucket combinations -are rejected. `PLINDER_MIRROR_URL` may select another HTTP(S) bucket root with -the same release layout and manifest. The local cache layout is unchanged. - -## Download and cache behavior - -The consumer uses cloudpathlib HTTP paths. A release-relative `manifest.jsonl.gz` -provides listing, sizes and base64 MD5 checksums. Directory traversal and file -transfers use cloudpathlib; the manifest is cached in-process. - -Transfers stream into temporary files, verify size and MD5, and atomically -replace the destination. Interrupted transfers retry from the start, with up to -three attempts. They do not resume at a byte offset. Failed validation preserves -an existing destination. Parallel batches use eight workers and propagate errors. - -Cached files with the expected size are reused without rehashing. Missing and -size-mismatched files are fetched again. Same-size local corruption requires a -forced refresh (`data.force_update=true`) or deletion of the affected file. -Refreshing a ZIP clears its extraction marker. `PLINDER_OFFLINE` bypasses all -network access when set to any nonempty value; unset it to return online. - -Google libraries are optional for consumers and available through `plinder[data]`. -The docs extra still declares Google storage because `docs/tablegen.py` uses it -for annotation-table generation. That docs download has not yet moved to R2. -Ingestion and generation compatibility are outside this static-release change. - -## Building the static manifest - -```bash -python scripts/build_r2_manifest.py INVENTORY_DIR OUTPUT.jsonl.gz -``` - -`INVENTORY_DIR/manifest-files.json` lists shard paths and SHA256 values. Each -compressed JSONL shard contains source rows with `name`, `size` and `md5Hash`. -The builder verifies shard checksums, rejects duplicate/invalid keys and writes -sorted release-relative records for `2024-06/v2`. It performs no cloud I/O. - -The existing public manifest lists 5,891 objects totaling 219,677,619,278 bytes. -Its 207,150 published bytes were checked on 2026-09-14 against SHA256 -`9b520375910b2e9bc7d258b0186e21488dc2236991976683a4bfa0ccbaa04e1b`. -The full download validation below also checked every object's size and MD5. - -## Validation - -```bash -python -m pytest --noconftest tests/core/test_r2.py tests/core/test_core_config.py tests/core/test_atoms.py::test_default_sequence_alignment -q -o addopts='' -``` - -30 focused tests passed with Python 3.12.13, cloudpathlib 0.25.0 and NumPy 2.5.3. -They cover directory/archive access, checksums, atomic replacement, cache reuse, -forced refresh, path containment, offline behavior, interrupted-transfer retry -and integer gap penalties required by Biotite's sequence alignment API. -`--noconftest` excludes repository-wide OpenStructure-dependent fixtures. These -results do not represent full project CI or evaluation coverage. - -On 2026-09-14, the revised implementation downloaded the complete release with -`plinder_download --release 2024-06 --iteration v2 --yes`: all 5,891 objects -(219,677,619,278 bytes) passed size and MD5 checks. All 1,060 system archives and -both linked-structure archives extracted successfully. Their contents total -567,902,199,349 bytes, so allow roughly 788 GB plus filesystem overhead for the -downloaded files and extracted contents together. - -A deliberate process interruption preserved 1,011 completed files; restarting -the CLI reused them without changing their timestamps. A final fully cached CLI -run exited successfully in 6.44 seconds, downloaded no dataset files and left all -5,891 files unchanged. No unplanned transfer errors or retries occurred during -the full download; interrupted-transfer retry was exercised by the HTTP test -server. A separate offline reuse check passed with networking blocked. - -The API tutorial, index-query notebook and system-access notebook executed all -63 code cells against the local release. A `PlinderSystem` structure smoke test -loaded the receptor and ligand for `4agi__1__1.C__1.W` successfully. - -Repository-wide Ruff checks, strict typing of all 91 source files and the full -Sphinx documentation build with warnings treated as errors passed locally. -The manifest builder was checked with synthetic inventories for sorting, -checksum/duplicate rejection and preservation of an existing output on those -validation failures. These checks do not represent full evaluation coverage. - -No dataset copy, permission change or public GCS retirement is part of these -commits. Existing installed clients still use GCS until upgraded. diff --git a/docs/tutorial/dataset.md b/docs/tutorial/dataset.md index 51ec10f31..514a4407a 100644 --- a/docs/tutorial/dataset.md +++ b/docs/tutorial/dataset.md @@ -3,31 +3,29 @@ The `plinder` Python package is the main entry point for downloading release files, querying annotations, and reconstructing structures. +PLINDER downloads the files needed by each API call automatically and reuses +them on subsequent calls. You can start exploring straight away. + ## Install PLINDER ```bash pip install plinder ``` -The published `2024-06/v2` release downloads from Cloudflare R2 -(`https://plinderdata.org`) with checksum verification and automatic retries; -no Google Cloud SDK is required. `PLINDER_MIRROR_URL` selects another HTTP(S) -mirror serving the same release layout and manifest. - -A release is identified by the month of its PDB snapshot and a number within -that month: +A release is identified by the month of its PDB snapshot. The current release +(`2026-09`) is the default; select another with: ```bash -export PLINDER_RELEASE=2026-07 -export PLINDER_RELEASE_NUMBER=1 +export PLINDER_RELEASE=2026-09 ``` ## Continue with a runnable guide -Start with {doc}`/examples/1_download`, then use -{doc}`/examples/2_query_filter_index` to select ligand or protein-interface +Start with {doc}`/examples/2_query_filter_index` to select ligand or protein-interface records. The remaining {doc}`/examples/index` guides cover coordinate access, similarity analysis, and custom scoring. +For a local copy to use offline, see {doc}`/examples/1_download`. + Use the {doc}`/dataset` page as the release and table reference, and {doc}`/api/index` for individual Python signatures. diff --git a/environment.yml b/environment.yml index e415115f7..51a580b61 100644 --- a/environment.yml +++ b/environment.yml @@ -15,9 +15,6 @@ channels: - bioconda dependencies: - python=3.12.* - - reduce - mmseqs2 - foldseek - bioconda::openstructure>=2.12.0 - - pip: - - keyrings.google-artifactregistry-auth==1.1.2 diff --git a/flows/docker.py b/flows/docker.py deleted file mode 100644 index ec38e2f81..000000000 --- a/flows/docker.py +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright (c) 2024, Plinder Development Team -# Distributed under the terms of the Apache License 2.0 -from __future__ import annotations - -import logging -from argparse import ArgumentParser -from os import environ -from pathlib import Path -from shlex import split -from shutil import which -from subprocess import PIPE, check_output -from sys import argv as sys_argv -from typing import List, Optional - -from proc import Proc - -logging.basicConfig(level=logging.INFO) -LOG = logging.getLogger("doc") -DEPENDENCY_BLOCKS = "data,test,eval,plots" - - -def get_dev_tag() -> str: - """ - Get the current git describe which includes base image tag - and whether or not the repository is in a dirty (uncommitted) - state. - - Returns - ------- - tag : str - current git describe - """ - return check_output( - [ - "git", - "describe", - "--tags", - "--always", - "--dirty", - "--abbrev=8", - ], - text=True, - ).strip() - - -def get_version_bump(base_tag: str | None = None) -> str: - """ - Inspect the git history for "bumpversion {phrase}" - and return the new version. - """ - if base_tag is None: - current = get_dev_tag() - base_tag = current.split("-")[0] - bump = "patch" - tokens = [ - "bumpversion major", - "bumpversion minor", - "bumpversion skip", - ] - for token in tokens: - log = check_output( - split( - " ".join( - [ - "git", - "log", - f"{base_tag}..HEAD", - "--oneline", - "--grep", - f"'{token}'", - "--format=%s", - ] - ) - ), - text=True, - ).strip() - # origin/main hack was to get around docs branch but isn'try: - # present before pushing the tag back to main so just search for PR number - if log and "(#" in log: - bump = token.split()[1] - break - if bump == "skip": - return "" - try: - import semver - - new_version = getattr(semver, f"bump_{bump}")(base_tag.lstrip("v")) - new_version = f"v{new_version}" - print(new_version) - return new_version - except ImportError: - LOG.error("could not import semver") - return "" - - -def get_image() -> str: - """ - Minimize the diff if any of this changes - """ - url = "ghcr.io" - organization = "plinder-org" - image_name = "plinder" - return f"{url}/{organization}/{image_name}" - - -def get_docker() -> str: - """ - Sanity check for docker in environment - """ - docker = which("docker") - if docker is None: - raise ValueError("could not find docker executable!") - if not (Path.cwd() / "docker-compose.yml").is_file(): - raise ValueError("could not find docker-compose.yml, are you in repo root?") - return docker - - -def get_env(tag: str | None = None) -> dict[str, str]: - """ - Get the build env that simulates docker compose - - Parameters - ---------- - tag : str, default=None - the tag to use instead of the current dev tag - - Returns - ------- - env : dict[str, str] - the build environment - """ - image_repo = "/".join(get_image().split("/")[:-1]) - dev_tag = get_dev_tag() - base_tag = dev_tag.split("-")[0] - build_tag = tag or dev_tag - LOG.info(f"get_env: base_tag={base_tag} build_tag={build_tag}") - return { - **environ, - **{ - "BASE_TAG": base_tag, - "IMAGE_REPO": image_repo, - "BUILD_TAG": build_tag, - "INDEX_URL": "https://pypi.org/simple", - }, - } - - -def pull_base_image(build: bool = False, promote: bool = False) -> None: - """ - Make sure the base image is up to date. Optionally - promote the image tag using semver by inspecting the - git history for "bumpverson {phrase}". - - Parameters - ---------- - promote : bool, default=False - if True, promote the image tag using semver - """ - docker = get_docker() - env = get_env() - cmd = [docker, "compose", "pull", "base", "--quiet"] - Proc(cmd, env=env).execute() - if build: - cmd = [ - docker, - "build", - "-f", - "dockerfiles/base/Dockerfile", - "-t", - f"{env['IMAGE_REPO']}/plinder-base:{env['BASE_TAG']}", - "--secret", - "id=INDEX_URL", - "--build-arg", - f"BASE_TAG={env['BASE_TAG']}", - "--build-arg", - f"DEPENDENCY_BLOCKS={DEPENDENCY_BLOCKS}", - (Path.cwd() / "dockerfiles/base").as_posix(), - ] - Proc(cmd, env=env).execute() - if promote: - promote_base_image(env) - - -def promote_base_image(env: dict[str, str]) -> None: - """ - Promote the base image - """ - base_tag = env["BASE_TAG"] - new_tag = get_version_bump(base_tag) - if not new_tag: - LOG.info("skipping base image promotion") - return - image = f"{env['IMAGE_REPO']}/plinder-base" - orig_image = f"{image}:{base_tag}" - new_image = f"{image}:{new_tag}" - LOG.info(f"promoting base image to {new_tag}") - docker = get_docker() - Proc([docker, "tag", orig_image, new_image], env=env).execute() - check_output(["git", "tag", new_tag]) - - -def build_image(tag: str | None = None, push: bool = False) -> str: - """ - Build the image - - Parameters - ---------- - tag : str, default=None - the tag to use instead of the current dev tag - push : bool, default=False - push the image to the artifact registry - - Returns - ------- - image : str - the full image including tag - """ - check_output(["git", "fetch", "--tags"]) - docker = get_docker() - env = get_env(tag) - image = f"{env['IMAGE_REPO']}/plinder" - build_tag = env["BUILD_TAG"] - registry = environ.get("PLINDER_REGISTRY", "/".join(get_image().split("/")[:-1])) - cmd = [ - docker, - "build", - "-f", - "dockerfiles/main/Dockerfile", - "-t", - f"{env['IMAGE_REPO']}/plinder:{build_tag}", - "-t", - f"{registry}/plinder:{build_tag}", - "--secret", - "id=INDEX_URL", - "--build-arg", - f"BASE_IMAGE={env['IMAGE_REPO']}/plinder-base", - "--build-arg", - f"BASE_TAG={env['BASE_TAG']}", - "--build-arg", - f"DEPENDENCY_BLOCKS={DEPENDENCY_BLOCKS}", - Path.cwd().as_posix(), - ] - Proc(cmd, env=env).execute() - if push: - cmd = [docker, "push", f"{registry}/plinder:{build_tag}"] - Proc(cmd, env=env).execute() - return f"{image}:{build_tag}" - - -def test_image( - tag: str, - push: bool = False, - args: Optional[List[str]] = None, - dirty: bool = False, -) -> None: - """ - Run the test service from docker compose. Optionally - push both the base and plinder images to the artifact - registry. - - Parameters - ---------- - tag : str - the tag to use instead of the current dev tag - push : bool, default=False - if True, push images to the artifact registry - args : List[str], default=None - the arguments to pass to the image - dirty : bool, default=False - if True, mount the current working tree - """ - env = get_env(tag) - docker = get_docker() - cmd = [docker, "compose", "run", "-e", "PLINDER_OFFLINE=true"] - if dirty: - cmd.extend(["-v", f"{Path.cwd()}/src/plinder:/opt/conda/lib/python3.9/site-packages/plinder"]) - cmd.append("test") - if args is not None and len(args): - cmd.extend( - split(f'''/bin/bash -c "python -m pytest -n auto -v {' '.join(args)} && cp .coverage reports/.coverage"''') - ) - Proc(cmd, env=env).execute() - if push: - cmd = [docker, "compose", "push", "base", "plinder", "--quiet"] - Proc(cmd, env=env).execute() - - -def run_image( - args: Optional[List[str]] = None, - build: bool = False, - tag: str | None = None, - dirty: bool = False, - it: bool = False, - script: str | None = None, -) -> None: - """ - Run the image mounting the current working tree. This can be - convenient for debugging during development. - - Parameters - ---------- - image : str - the image to run - args : List[str], default=None - the arguments to pass to the image - """ - tag = tag or "latest" - image = build_image(tag=tag) if build else get_image() + ":" + tag - docker = which("docker") - if docker is None: - raise ValueError("could not find docker executable!") - home = Path.home() - host = Path.cwd() / "src" - guest = "/opt/conda/lib/python3.10/site-packages/plinder" - app = "/app/src" - if not host.is_dir(): - raise RuntimeError(f"could not find {host}, please run from repo root") - cmd = [ - docker, - "run", - "--platform", - "linux/amd64", - "-e", - "PLINDER_TEST_BASE_DIR=/app", - "-v", - f"{host.parent}/tests:/app/tests/", - "-v", - f"{host.parent}/column_descriptions:/app/column_descriptions/", - "-e", - "PLINDER_MOUNT=/plinder", - "-e", - "PLINDER_LOG_LEVEL=10", - "-v", - f"{home}/.local/share/plinder:/plinder", - "-v", - f"{host}/plinder:{guest}", - "-v", - f"{host}:{app}", - ] - if script: - cmd.extend(["-v", f"{host.parent}/{script}:{Path(app).parent}/{script}"]) - if it: - import pty - - pty.spawn(cmd + ["-it", image, "/bin/bash"]) - return - cmd.append(image) - if args is not None: - cmd.extend(args) - Proc(cmd).execute() - - -def main(argv: Optional[List[str]] = None): - """ - The main entrypoint for the docker helper - - Parameters - ---------- - argv : List[str], default=None - the command line arguments - """ - if argv is None: - argv = sys_argv[1:] - parser = ArgumentParser(prog="plinder docker helper") - subs = parser.add_subparsers(help="subcommand", dest="command") - subs.add_parser("bump", help="Get version bump") - pull = subs.add_parser("pull", help="Pull the base image") - pull.add_argument( - "--promote", - default=False, - action="store_true", - help="Promote the image tag using semver", - ) - build = subs.add_parser("build", help="Build the app image") - test = subs.add_parser("test", help="Test the app image") - test.add_argument("--dirty", default=False, action="store_true", help="Mount current working tree") - run = subs.add_parser("run", help="Run the app image") - run.add_argument("--it", default=False, action="store_true", help="Run in interactive mode") - run.add_argument("--script", default="", help="Script to run") - run.add_argument("--dirty", default=False, action="store_true", help="Mount current working tree") - for sub in [build, test, run]: - sub.add_argument( - "--tag", default=None, help="The image tag to pass to build_image", - ) - for sub in [build, test]: - sub.add_argument( - "--push", - default=False, - action="store_true", - help="Push the image to the artifact registry", - ) - for sub in [pull, run]: - sub.add_argument( - "--build", - default=False, - action="store_true", - help="Build the image before running", - ) - ns, args = parser.parse_known_args() - nsargs = vars(ns) - if len(args): - nsargs["args"] = args - command = nsargs.pop("command") - func = { - "bump": get_version_bump, - "pull": pull_base_image, - "build": build_image, - "test": test_image, - "run": run_image, - } - kwargs = {} if command == "bump" else nsargs - if command is None: - parser.print_help() - exit() - func[command](**kwargs) - - -if __name__ == "__main__": - main() diff --git a/flows/proc.py b/flows/proc.py deleted file mode 100644 index 1c4b74594..000000000 --- a/flows/proc.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2024, Plinder Development Team -# Distributed under the terms of the Apache License 2.0 -import io -import logging -from subprocess import PIPE, STDOUT, Popen -from typing import Any, Dict, List, Optional, Union - -logging.basicConfig(level=logging.INFO) -LOG = logging.getLogger("proc") - - -class Proc(Popen): - """ - An opinionated Popen with an execute method - to mirror communicate that handles input as strings, - supports early return, and logs as the process runs. - Also retain stdout and stderr as a list of strings - for downstream access. - """ - - def __init__( - self, - *args, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, - text=True, - **kwargs, - ): - super().__init__( - *args, - stdin=stdin, - stdout=stdout, - stderr=stderr, - text=text, - **kwargs, - ) - self._stderr = stderr - # squelch the static type checkers that - # see these pipes as None since we're using - # PIPEs for everything - self.stdin: io.TextIOWrapper - self.stdout: io.TextIOWrapper - self.stderr: io.TextIOWrapper - - def execute( - self, - *, - inputs: Optional[str] = None, - ) -> None: - """ - Like Popen.communicate but tail output - - Parameters - ---------- - inputs : str - a command to pass to the subprocess - - """ - if isinstance(self.args, list): - LOG.info(" ".join(map(str, self.args))) - else: - LOG.info(self.args) - - streams = [("stdout", "stdout"), ("stderr", "stderr")] - if self._stderr == STDOUT: - streams = [("stdout", "stream")] - with self: - try: - if inputs is not None: - self.stdin.write(inputs) - self.stdin.close() - for attr, stream in streams: - for line in getattr(self, attr): - LOG.info(f"{stream}: {line.rstrip()}") - except Exception: - self.kill() diff --git a/pyproject.toml b/pyproject.toml index 08d7bf25a..bfef683e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "rdkit>=2025.09.1", "pyarrow", "omegaconf", - "duckdb", + "duckdb>=1.5", "cloudpathlib>=0.25,<0.26", ] description = "PLINDER: Protein & Ligand INteraction Dataset and Evaluation Resource" @@ -32,6 +32,7 @@ Issues = "https://github.com/plinder-org/plinder/issues" [project.scripts] plinder_download = "plinder.core.index.utils:download_plinder_cmd" plinder_eval = "plinder.eval.cli:main" +plinder_prepare_release = "plinder.data.pipeline.public_release:main" [project.optional-dependencies] data = ["google-cloud-storage", "gcsfs", "numba"] @@ -87,15 +88,16 @@ docs = [ "matplotlib", ] -# Not published to PyPI; installed by default with `uv sync`. -# Mirrors requirements_data.txt, which remains for pip-based installs. +# Not published to PyPI (includes git-only packages); installed by default with +# `uv sync`. This is the complete development and CI environment. [dependency-groups] dev = [ - "plinder[dev,eval,loader]", + "plinder[dev,eval,loader,plots]", "networkit>11.0", "tabulate", "pdb-validation", "mmpdb", + "semver", ] [tool.uv.sources] @@ -103,6 +105,10 @@ pdb-validation = { git = "https://git.scicore.unibas.ch/schwede/ligand-validatio mmpdb = { git = "https://github.com/rdkit/mmpdb.git" } torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }] +[tool.uv] +# networkit source builds need CMake; require a published wheel. +no-build-package = ["networkit"] + [[tool.uv.index]] name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" diff --git a/requirements_data.txt b/requirements_data.txt deleted file mode 100644 index 0b7aa95fa..000000000 --- a/requirements_data.txt +++ /dev/null @@ -1,7 +0,0 @@ ---only-binary=networkit - networkit > 11.0 - tabulate - pdb-validation @ git+https://git.scicore.unibas.ch/schwede/ligand-validation.git - mmpdb @ git+https://github.com/rdkit/mmpdb.git - torch @ https://download.pytorch.org/whl/cpu/torch-2.5.1%2Bcpu-cp312-cp312-linux_x86_64.whl#sha256=4856f9d6925121d13c2df07aa7580b767f449dfe71ae5acde9c27535d5da4840 ; sys_platform == "linux" - torch >= 2.5 ; sys_platform == "darwin" diff --git a/scripts/build_r2_manifest.py b/scripts/build_r2_manifest.py deleted file mode 100644 index 6627a57cc..000000000 --- a/scripts/build_r2_manifest.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Build the release's HTTP listing from audited GCS metadata, without cloud I/O.""" - -import argparse -import base64 -import gzip -import hashlib -import json -from pathlib import Path - - -def build(inventory: Path, destination: Path) -> int: - shards = json.loads((inventory / "manifest-files.json").read_text()) - records = {} - for shard in shards: - path = (inventory / shard["path"]).resolve() - if not path.is_relative_to(inventory.resolve()): - raise ValueError("Invalid inventory path") - if hashlib.sha256(path.read_bytes()).hexdigest() != shard["sha256"]: - raise ValueError("Inventory checksum mismatch") - with gzip.open(path, "rt") as stream: - for line in stream: - r = json.loads(line) - if not r["name"].startswith("2024-06/v2/"): - raise ValueError("Unexpected release") - key = r["name"][len("2024-06/v2/") :] - if key.endswith("/"): - continue - if ( - not key - or "\\" in key - or any(p in {"", ".", ".."} for p in key.split("/")) - or key in records - ): - raise ValueError("Invalid or duplicate object key") - size = int(r["size"]) - if size < 0 or len(base64.b64decode(r["md5Hash"], validate=True)) != 16: - raise ValueError("Invalid object metadata") - records[key] = {"key": key, "size": size, "md5": r["md5Hash"]} - destination.parent.mkdir(parents=True, exist_ok=True) - with gzip.open(destination, "wt") as stream: - for key in sorted(records): - stream.write(json.dumps(records[key], separators=(",", ":")) + "\n") - return len(records) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("inventory", type=Path) - parser.add_argument("destination", type=Path) - args = parser.parse_args() - print(build(args.inventory, args.destination)) diff --git a/scripts/slurm/README.md b/scripts/slurm/README.md index bd445c954..b4b5283d3 100644 --- a/scripts/slurm/README.md +++ b/scripts/slurm/README.md @@ -619,3 +619,24 @@ the validated alignment shards under `alignment_cigars/search_db=holo/alignment_type=*/shard=*.parquet`. Per-PDB raw search files and the other derived score datasets are generation intermediates; mapped per-PDB files exist only transiently on node-local scratch. + +## Separate the public release from working data + +Keep the completed ingest as the working directory. Build a sibling directory +containing only the files used by the release API: + +```bash +python -m plinder.data.pipeline.public_release \ + /path/to/ingest_2026-09 /path/to/PLINDER-2026-09 +``` + +The public directory includes the index tables, similarity tables, cover tables, +ligand archives and scores, alignment CIGARs, and MMseqs/Foldseek search databases. +It excludes raw alignments, score caches, scratch files, STEAM databases, and JSON +files containing local paths. By default the files are hard-linked, so this adds +almost no data storage but is not an independent copy: do not edit either tree's +files in place. Pass `--copy` for an independent copy. After a working-release +repair, rerun with `--replace` to refresh the public tree. + +Upload the public directory, not the working ingest. The Foldseek databases +contain internal symlinks, so use `--no-ignore-symlinks` when syncing to GCS. diff --git a/src/plinder/core/__init__.py b/src/plinder/core/__init__.py index c4b1e8338..f38fbe9ad 100644 --- a/src/plinder/core/__init__.py +++ b/src/plinder/core/__init__.py @@ -4,10 +4,11 @@ The plinder.core package collects useful functions and classes for interacting with the PLINDER dataset. It manages app configuration and will automatically download (and / or sync) the dataset to a local cache in a lazy manner, when -particular assets are requested. Downloads are checked against the release -manifest for size and MD5 before replacing a local file. Cached files with the -expected size are reused without rehashing; force a refresh to repair same-size -local corruption. +particular assets are requested. Downloads come directly from the Cameo file +server and are checked against HTTP Content-Length before replacing a local +file. Cached files with matching size and Last-Modified time are reused; server +hotfixes are fetched on the next access. Force a refresh to repair local +corruption that preserves both size and modification time. Note ---- diff --git a/src/plinder/core/index/utils.py b/src/plinder/core/index/utils.py index 6455d7350..f5a1aed02 100644 --- a/src/plinder/core/index/utils.py +++ b/src/plinder/core/index/utils.py @@ -31,6 +31,7 @@ "complete similarity exports", ( "protein_similarity_scores", + "monomer_similarity_scores", "ligand_similarity_scores", "interface_similarity_scores", "interface_half_similarity_scores", diff --git a/src/plinder/core/release.py b/src/plinder/core/release.py index d825eda9b..648385cf1 100644 --- a/src/plinder/core/release.py +++ b/src/plinder/core/release.py @@ -70,16 +70,23 @@ def _release_file( ), "ligand_archives": "ligand_archives", "ligand_archive": "ligand_archives/{shard}.parquet", - "ligand_archives_manifest": "ligand_archives/manifest.json", "ligand_scores": "ligand_scores", "ligand_sampling": "ligand_sampling", "interface_sampling": "interface_sampling", "search_databases": "search_databases", "search_database": "search_databases/holo_{backend}", - "search_databases_manifest": "search_databases/manifest.json", "search_database_overlay": "search_databases/weekly_delta/holo_{backend}", + "monomer_search_database": "search_databases/monomer_{backend}", + "monomer_search_database_overlay": ( + "search_databases/weekly_delta/monomer_{backend}" + ), "search_database_shadowed_entries": "search_databases/shadowed_entries.parquet", "protein_similarity_scores": "exports/protein_similarity_scores", + "protein_similarity_score_shard": ( + "exports/protein_similarity_scores/alignment_type={alignment_type}/" + "shard={shard}.parquet" + ), + "monomer_similarity_scores": "exports/monomer_similarity_scores", "ligand_similarity_scores": "exports/ligand_similarity_scores.parquet", "interface_similarity_scores": "exports/interface_similarity_scores.parquet", "interface_half_similarity_scores": ( diff --git a/src/plinder/core/scores/custom.py b/src/plinder/core/scores/custom.py index 26d424f56..7492e8eea 100644 --- a/src/plinder/core/scores/custom.py +++ b/src/plinder/core/scores/custom.py @@ -4,7 +4,6 @@ from __future__ import annotations -import json import logging import re import shutil @@ -32,12 +31,6 @@ CIF_SEARCH_BACKENDS = ("foldseek", "mmseqs") SEQUENCE_SEARCH_BACKENDS = ("mmseqs",) SEARCH_BACKENDS = CIF_SEARCH_BACKENDS -EXACT_CLUSTER_CONTRACT: Mapping[str, object] = { - "identity": 1.0, - "coverage": 1.0, - "coverage_mode": 0, - "compressed_search_target": False, -} @dataclass(frozen=True) @@ -62,9 +55,15 @@ class CustomScoringAssets: alignment_chain_lookup: Path search_databases: Mapping[str, SearchDatabaseBundle] ligand_archives: Mapping[str, Path] + monomer_search_databases: Mapping[str, SearchDatabaseBundle] = field( + default_factory=dict + ) overlay_search_databases: Mapping[str, SearchDatabaseBundle] = field( default_factory=dict ) + overlay_monomer_search_databases: Mapping[str, SearchDatabaseBundle] = field( + default_factory=dict + ) shadowed_entries: frozenset[str] = frozenset() @@ -153,6 +152,7 @@ class CustomScoringResult: query_inputs: CustomQueryInputs query_databases: CustomQueryDatabases protein_hits: Mapping[str, Path] + chain_similarity_scores: Path score_alignments: Mapping[str, Path] protein_score_alignments: Mapping[str, Path] protein_scores: Path @@ -168,6 +168,7 @@ class CustomSequenceScoringResult: query_inputs: CustomQueryInputs query_databases: CustomQueryDatabases protein_hits: Mapping[str, Path] + chain_similarity_scores: Path protein_score_alignments: Mapping[str, Path] protein_scores: Path aligned_pocket_residues: Path | None @@ -182,15 +183,6 @@ class _SequenceEntry: pdb_id: str -def _manifest_member(root: Path, value: object, *, field: str) -> Path: - if not isinstance(value, str) or not value: - raise ValueError(f"invalid {field} in {root / 'exact_cluster.json'}") - relative = Path(value) - if relative.is_absolute() or relative.name != value: - raise ValueError(f"unsafe {field} in {root / 'exact_cluster.json'}: {value}") - return root / relative - - def _require_database_prefix( prefix: Path, *, @@ -219,12 +211,16 @@ def _validate_portable_links(root: Path) -> None: raise ValueError(f"non-portable database link: {path} -> {path.readlink()}") -def _local_search_database_root(data_dir: Path, backend: str) -> Path: +def _local_search_database_root( + data_dir: Path, backend: str, *, monomer: bool = False +) -> Path: """Resolve either a published bundle or an unmodified ingest output.""" - published = PlinderRelease(data_dir).path("search_database", backend=backend) + artifact = "monomer_search_database" if monomer else "search_database" + prefix = "monomer" if monomer else "holo" + published = PlinderRelease(data_dir).path(artifact, backend=backend) if published.is_dir(): return published - ingest = data_dir / "dbs" / "subdbs" / f"holo_{backend}" + ingest = data_dir / "dbs" / "subdbs" / f"{prefix}_{backend}" return ingest if ingest.is_dir() else published @@ -232,67 +228,35 @@ def resolve_search_database( backend: str, *, data_dir: Path | None = None, + monomer: bool = False, ) -> SearchDatabaseBundle: """Download or validate one portable PLINDER search database bundle.""" if backend not in SEARCH_BACKENDS: raise ValueError( f"unsupported search backend {backend!r}; expected one of {SEARCH_BACKENDS}" ) + artifact = "monomer_search_database" if monomer else "search_database" if data_dir is None: root = _release_file( - "search_database", + artifact, data_dir=None, description=f"{backend} search database", backend=backend, ) else: - root = _local_search_database_root(Path(data_dir), backend) + root = _local_search_database_root(Path(data_dir), backend, monomer=monomer) return _validate_search_database(backend, Path(root)) def _validate_search_database(backend: str, root: Path) -> SearchDatabaseBundle: root = Path(root) - manifest_path = _require_file( - root / "exact_cluster.json", - description=f"{backend} search database manifest", - ) - try: - manifest = json.loads(manifest_path.read_text()) - except (OSError, json.JSONDecodeError) as exc: - raise ValueError(f"invalid search database manifest: {manifest_path}") from exc - if not isinstance(manifest, dict): - raise ValueError(f"invalid search database manifest: {manifest_path}") - if manifest.get("alignment_type") != backend: - raise ValueError( - f"search database backend mismatch in {manifest_path}: " - f"{manifest.get('alignment_type')!r}" - ) - if manifest.get("portable") is not True: - raise ValueError(f"search database is not portable: {manifest_path}") - mismatches = { - key: manifest.get(key) - for key, expected in EXACT_CLUSTER_CONTRACT.items() - if manifest.get(key) != expected - } - if mismatches: - raise ValueError( - f"search database does not satisfy the exact-cluster contract in " - f"{manifest_path}: {mismatches}" - ) - - search_target = _manifest_member( - root, manifest.get("search_target"), field="search_target" - ) - conversion_target = _manifest_member( - root, manifest.get("conversion_target"), field="conversion_target" - ) - cluster_alignments = None - if backend == "mmseqs": - cluster_alignments = _manifest_member( - root, - manifest.get("cluster_alignments"), - field="cluster_alignments", - ) + if backend == "foldseek": + search_target = conversion_target = root / "clustered" + cluster_alignments = None + else: + search_target = root / "representatives" + conversion_target = root / root.name + cluster_alignments = root / "cluster_alignments" _require_database_prefix( search_target, @@ -315,7 +279,7 @@ def _validate_search_database(backend: str, root: Path) -> SearchDatabaseBundle: search_target=search_target, conversion_target=conversion_target, cluster_alignments=cluster_alignments, - manifest=manifest, + manifest={}, ) @@ -349,6 +313,7 @@ def resolve_custom_scoring_assets( data_dir: Path | None = None, backends: Iterable[str] = CIF_SEARCH_BACKENDS, ligand_pdb_ids: Iterable[str] = (), + include_monomers: bool = False, ) -> CustomScoringAssets: """Download or validate the bounded asset set for custom scoring. @@ -376,34 +341,48 @@ def resolve_custom_scoring_assets( backend: resolve_search_database(backend, data_dir=data_dir) for backend in selected_backends } + monomer_databases = {} + for backend in selected_backends if include_monomers else (): + try: + monomer_databases[backend] = resolve_search_database( + backend, data_dir=data_dir, monomer=True + ) + except FileNotFoundError as exc: + # A release or ingest may have no monomer universe for a backend. + LOG.warning(f"no {backend} monomer search database; skipping: {exc}") release = PlinderRelease(data_dir) + overlay_databases = {} + overlay_monomer_databases = {} + for backend in selected_backends: + try: + overlay = release.fetch("search_database_overlay", backend=backend) + except FileNotFoundError: + pass + else: + overlay_databases[backend] = _validate_search_database(backend, overlay) + if include_monomers: + try: + monomer_overlay = release.fetch( + "monomer_search_database_overlay", backend=backend + ) + except FileNotFoundError: + pass + else: + overlay_monomer_databases[backend] = _validate_search_database( + backend, monomer_overlay + ) + shadowed_entries: frozenset[str] = frozenset() try: - manifest_path = release.fetch("search_databases_manifest") + shadowed_path = release.fetch("search_database_shadowed_entries") except FileNotFoundError: - manifest = {} + if overlay_databases or overlay_monomer_databases: + raise ValueError( + "search database overlay is missing shadowed entries" + ) from None else: - manifest = json.loads(manifest_path.read_text()) - if not isinstance(manifest, dict): - raise ValueError(f"invalid search database manifest: {manifest_path}") - overlays = manifest.get("overlays", {}) - if not isinstance(overlays, dict): - raise ValueError("invalid search database overlay list") - overlay_databases = { - backend: _validate_search_database( - backend, - release.fetch("search_database_overlay", backend=backend), - ) - for backend in selected_backends - if backend in overlays - } - shadowed_entries: frozenset[str] = frozenset() - if manifest.get("shadowed_entries"): - shadowed_path = release.fetch("search_database_shadowed_entries") shadowed_entries = frozenset( pd.read_parquet(shadowed_path, columns=["pdb_id"])["pdb_id"].astype(str) ) - if overlay_databases and not shadowed_entries: - raise ValueError("search database overlay is missing its shadowed entry list") ligand_archives = resolve_ligand_archives( ligand_pdb_ids, data_dir=data_dir, @@ -415,7 +394,9 @@ def resolve_custom_scoring_assets( alignment_chain_lookup=resolved_index["alignment_chain_lookup"], search_databases=databases, ligand_archives=ligand_archives, + monomer_search_databases=monomer_databases, overlay_search_databases=overlay_databases, + overlay_monomer_search_databases=overlay_monomer_databases, shadowed_entries=shadowed_entries, ) @@ -1129,8 +1110,9 @@ def _build_mmseqs_target_subset( entry_ids: Iterable[str], output_dir: Path, threads: int = 1, -) -> SearchDatabaseBundle: - """Build an MMseqs target from scoreable chains in selected entries.""" + monomer_only: bool = False, +) -> SearchDatabaseBundle | None: + """Build a bounded MMseqs target from selected release chains.""" selected = _plinder_entry_subset(entry_ids) assert selected is not None if threads < 1: @@ -1147,6 +1129,7 @@ def _build_mmseqs_target_subset( "chain_receptor_type", "chain_is_holo", "chain_sequence", + *(["chain_is_ligand_like"] if monomer_only else []), ], filters=[("entry_pdb_id", "in", list(selected))], ) @@ -1175,18 +1158,25 @@ def _build_mmseqs_target_subset( how="left", validate="one_to_one", ) + scoreable = chains["chain_is_holo"].fillna(False).astype(bool) | chains[ + "chain_is_interface" + ].eq(True) + selected_chains = ( + ~scoreable & ~chains["chain_is_ligand_like"].fillna(False).astype(bool) + if monomer_only + else scoreable + ) chains = chains.loc[ chains["chain_receptor_type"].fillna("").astype(str).eq("protein") - & ( - chains["chain_is_holo"].fillna(False).astype(bool) - | chains["chain_is_interface"].eq(True) - ) + & selected_chains & chains["chain_auth_id"].notna() & chains["chain_sequence"].notna() ].copy() + if monomer_only and chains.empty: + return None found = set(chains["entry_pdb_id"].astype(str)) missing = sorted(set(selected).difference(found)) - if missing: + if missing and not monomer_only: raise ValueError( "selected PLINDER entries have no scoreable receptor or interface " f"chains: {missing[:10]}" @@ -1248,12 +1238,14 @@ def _resolve_workflow_assets( plinder_entry_ids: tuple[str, ...] | None, work_dir: Path, threads: int, + include_monomers: bool = False, ) -> CustomScoringAssets: if plinder_entry_ids is not None and backends != ("mmseqs",): raise ValueError("plinder_entry_ids currently requires backends=('mmseqs',)") assets = resolve_custom_scoring_assets( data_dir=data_dir, backends=() if plinder_entry_ids is not None else backends, + include_monomers=include_monomers and plinder_entry_ids is None, ) if plinder_entry_ids is None: return assets @@ -1264,7 +1256,27 @@ def _resolve_workflow_assets( output_dir=work_dir / "plinder_target_subset", threads=threads, ) - return replace(assets, search_databases={"mmseqs": subset}) + assert subset is not None + monomer_subset = ( + _build_mmseqs_target_subset( + assets.entry_chains, + assets.interface_annotations, + entry_ids=plinder_entry_ids, + output_dir=work_dir / "plinder_monomer_subset", + threads=threads, + monomer_only=True, + ) + if include_monomers + else None + ) + return replace( + assets, + search_databases={"mmseqs": subset}, + monomer_search_databases=( + {"mmseqs": monomer_subset} if monomer_subset is not None else {} + ), + shadowed_entries=frozenset(), + ) def _parse_plinder_target_identifier( @@ -1938,6 +1950,7 @@ def prepare_custom_protein_score_alignments( columns=[ "qaln", "taln", + "cigar", "target_selected_residue_indices", ], errors="ignore", @@ -2067,6 +2080,122 @@ def run_custom_protein_searches( return outputs +def run_custom_monomer_searches( + *, + query_databases: CustomQueryDatabases, + assets: CustomScoringAssets, + output_dir: Path, + scratch_dir: Path, + config: CustomProteinSearchConfig | None, + backends: Iterable[str], + threads: int, +) -> dict[str, Path]: + """Search protein chains absent from pocket/interface targets.""" + if not assets.monomer_search_databases: + return {} + monomer_assets = replace( + assets, + search_databases=assets.monomer_search_databases, + overlay_search_databases=assets.overlay_monomer_search_databases, + ) + return run_custom_protein_searches( + query_databases=query_databases, + assets=monomer_assets, + output_dir=output_dir, + scratch_dir=scratch_dir, + config=config, + backends=( + backend + for backend in backends + if backend in assets.monomer_search_databases + ), + threads=threads, + ) + + +def write_custom_chain_similarity_scores( + protein_hits: Mapping[str, Path], + *, + output_path: Path, + chain_manifest: Path | None = None, + monomer_hits: Mapping[str, Path] | None = None, +) -> Path: + """Write direct custom-to-PLINDER protein-chain alignment scores. + + Coverage and similarity values use integer percentages, as in the release + protein similarity table. Each alignment remains a separate row. + """ + from plinder.data.annotations.get_similarity_scores import ( + get_sequence_similarity_helper, + ) + + columns = [ + "sequence_id", + "query_id", + "query_chain_id", + "structure_id", + "query_chain_asym_id", + "target_entry", + "target_chain_asym_id", + "source", + "qcov", + "tcov", + "fident", + "seqsim", + "lddt", + ] + frames = [] + for backend, path in [*protein_hits.items(), *(monomer_hits or {}).items()]: + hits = pd.read_parquet(path) + if hits.empty: + continue + required = set(columns).difference({"sequence_id", "seqsim", "lddt"}) | { + "qaln", + "taln", + } + if backend == "foldseek": + required.add("lddt") + missing = sorted(required.difference(hits.columns)) + if missing: + raise ValueError(f"{backend} custom hit table is missing columns {missing}") + scores = hits[[column for column in columns if column in hits]].copy() + scores["seqsim"] = [ + get_sequence_similarity_helper(str(query).upper(), str(target).upper()) + for query, target in zip(hits["qaln"], hits["taln"], strict=True) + ] + if "lddt" not in scores: + scores["lddt"] = pd.NA + for column in ("qcov", "tcov", "fident", "seqsim", "lddt"): + scores[column] = ( + np.floor(pd.to_numeric(scores[column], errors="coerce") * 100 + 0.5) + .clip(0, 100) + .astype("UInt8") + ) + frames.append(scores) + result = ( + pd.concat(frames, ignore_index=True) + if frames + else pd.DataFrame(columns=columns) + ) + if chain_manifest is not None: + manifest = pd.read_parquet(chain_manifest, columns=["query_id", "sequence_id"]) + if manifest["query_id"].duplicated().any(): + raise ValueError("query chain manifest contains duplicate query IDs") + result = result.drop(columns="sequence_id", errors="ignore") + result = result.merge( + manifest, on="query_id", how="left", validate="many_to_one" + ) + if result["sequence_id"].isna().any(): + raise ValueError("custom chain scores contain unknown query IDs") + else: + result["sequence_id"] = pd.NA + result = result[columns] + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + result.to_parquet(output_path, index=False, compression="zstd") + return output_path + + def _target_entry_ids(score_alignments: Mapping[str, Path]) -> set[str]: target_ids: set[str] = set() for path in score_alignments.values(): @@ -2674,6 +2803,7 @@ def score_custom_sequence_file( plinder_entry_ids: Iterable[str] | None = None, threads: int = 1, store_aligned_pocket_residues: bool = False, + include_monomers: bool = False, ) -> CustomSequenceScoringResult: """Search protein sequences and score PLINDER ligand-pocket identity. @@ -2710,6 +2840,7 @@ def score_custom_sequence_file( plinder_entry_ids=selected_plinder_entries, work_dir=work_dir, threads=threads, + include_monomers=include_monomers, ) protein_hits = run_custom_protein_searches( query_databases=query_databases, @@ -2722,6 +2853,24 @@ def score_custom_sequence_file( backends=selected_backends, threads=threads, ) + monomer_hits = run_custom_monomer_searches( + query_databases=query_databases, + assets=assets, + output_dir=work_dir / "monomer_hits", + scratch_dir=( + Path(scratch_dir) if scratch_dir is not None else work_dir / "scratch" + ) + / "monomer", + config=search_config, + backends=selected_backends, + threads=threads, + ) + chain_similarity_scores = write_custom_chain_similarity_scores( + protein_hits, + output_path=work_dir / "chain_similarity_scores.parquet", + chain_manifest=query_inputs.chain_manifest, + monomer_hits=monomer_hits, + ) manifest = pd.read_parquet(query_inputs.chain_manifest) entries_by_structure = { str(structure_id): _SequenceEntry(pdb_id=str(structure_id)) @@ -2766,6 +2915,7 @@ def score_custom_sequence_file( query_inputs=query_inputs, query_databases=query_databases, protein_hits=protein_hits, + chain_similarity_scores=chain_similarity_scores, protein_score_alignments=protein_score_alignments, protein_scores=protein_score_path, aligned_pocket_residues=aligned_pocket_residue_path, @@ -2940,6 +3090,7 @@ def score_custom_cif_files( threads: int = 1, shape_score_threads: int = 1, store_aligned_pocket_residues: bool = False, + include_monomers: bool = False, ) -> CustomScoringResult: """Run custom-CIF annotation, protein search, and PLINDER scoring. @@ -3008,6 +3159,7 @@ def score_custom_cif_files( plinder_entry_ids=selected_plinder_entries, work_dir=work_dir, threads=threads, + include_monomers=include_monomers, ) protein_hits = run_custom_protein_searches( query_databases=query_databases, @@ -3020,6 +3172,23 @@ def score_custom_cif_files( backends=selected_backends, threads=threads, ) + monomer_hits = run_custom_monomer_searches( + query_databases=query_databases, + assets=assets, + output_dir=work_dir / "monomer_hits", + scratch_dir=( + Path(scratch_dir) if scratch_dir is not None else work_dir / "scratch" + ) + / "monomer", + config=search_config, + backends=selected_backends, + threads=threads, + ) + chain_similarity_scores = write_custom_chain_similarity_scores( + protein_hits, + output_path=work_dir / "chain_similarity_scores.parquet", + monomer_hits=monomer_hits, + ) score_alignments = prepare_custom_score_alignments( protein_hits, entries_by_structure=annotations.entries_by_structure, @@ -3091,6 +3260,7 @@ def score_custom_cif_files( query_inputs=query_inputs, query_databases=query_databases, protein_hits=protein_hits, + chain_similarity_scores=chain_similarity_scores, score_alignments=score_alignments, protein_score_alignments=protein_score_alignments, protein_scores=protein_score_path, diff --git a/src/plinder/core/scores/protein.py b/src/plinder/core/scores/protein.py index 7e8b1a7cb..e10796542 100644 --- a/src/plinder/core/scores/protein.py +++ b/src/plinder/core/scores/protein.py @@ -39,10 +39,21 @@ def query_protein_similarity( columns: list[str] | None = None, filters: Filters = None, release: PlinderRelease | None = None, + include_monomers: bool = False, ) -> pd.DataFrame: - """Query chain-level Foldseek and MMseqs similarities in integer percent.""" - dataset = (release or PlinderRelease()).fetch("protein_similarity_scores") - return read_score_table(dataset, columns=columns, filters=filters) + """Query chain-level Foldseek and MMseqs similarities in integer percent. + + Set ``include_monomers`` to include queries from protein chains outside + ligand pockets and protein interfaces. + """ + selected_release = release or PlinderRelease() + paths = [selected_release.fetch("protein_similarity_scores")] + if include_monomers: + paths.append(selected_release.fetch("monomer_similarity_scores")) + return pd.concat( + [read_score_table(path, columns=columns, filters=filters) for path in paths], + ignore_index=True, + ) def _selected_residue_positions( diff --git a/src/plinder/core/scores/reconstruct.py b/src/plinder/core/scores/reconstruct.py index 7ec468faf..4068f73d2 100644 --- a/src/plinder/core/scores/reconstruct.py +++ b/src/plinder/core/scores/reconstruct.py @@ -44,11 +44,15 @@ def prefetch_similarity_alignments( *, search_db: str = "holo", data_dir: Path | None = None, + include_protein_scores: bool = False, ) -> dict[str, dict[str, Path]]: """Download or verify only the mapped alignments needed by query systems. Calling this on an online node populates the normal PLINDER cache. Calling it later with offline mode enabled only checks those same local files. + With ``include_protein_scores``, each backend also needs its matching + protein-similarity export shard, which supplies the alignment scores that + the CIGAR shards omit (needed for ligand similarity reconstruction). """ if search_db != "holo": raise NotImplementedError( @@ -79,6 +83,17 @@ def prefetch_similarity_alignments( alignment_type=alignment_type, shard=shard, ) + if include_protein_scores: + _release_file( + "protein_similarity_score_shard", + data_dir=data_dir, + description=( + f"{alignment_type} protein similarity score " + f"shard {shard}" + ), + alignment_type=alignment_type, + shard=shard, + ) except FileNotFoundError as exc: resolved[key] = None resolution_errors[key] = exc @@ -458,6 +473,7 @@ def reconstruct_similarity_scores( query_systems, search_db=search_db, data_dir=data_dir, + include_protein_scores=True, ) selected_pdb_ids = { _pdb_id(system_id) for system_id in query_systems | target_systems diff --git a/src/plinder/core/scores/search.py b/src/plinder/core/scores/search.py index 564de5f0b..9e845e18a 100644 --- a/src/plinder/core/scores/search.py +++ b/src/plinder/core/scores/search.py @@ -36,6 +36,7 @@ def search( search_config: CustomProteinSearchConfig | None = None, plinder_entry_ids: Iterable[str] | None = None, store_aligned_pocket_residues: bool = False, + include_monomers: bool = True, ) -> CustomScoringResult | CustomSequenceScoringResult: """Search with the same structure table accepted by evaluation. @@ -54,6 +55,12 @@ def search( other PDB components are omitted. Supply ligand poses through ``ligand_path``. The returned result contains paths to score tables and alignment files. + ``chain_similarity_scores`` gives direct protein-chain hits, with query and + target coverage, identity, sequence similarity, and Foldseek lDDT as + integer percentages. FASTA results include the original ``sequence_id``. + With ``include_monomers=True``, chain hits also cover proteins outside + ligand pockets and protein interfaces. Pocket and interface scores still + use their own targets. ``search_config`` controls search filters. Its shared ``evalue`` defaults to 0.01; ``foldseek_evalue``, ``mmseqs_evalue`` and ``steam_evalue`` override it independently for their respective backends. @@ -70,6 +77,7 @@ def search( search_config=search_config, plinder_entry_ids=plinder_entry_ids, store_aligned_pocket_residues=store_aligned_pocket_residues, + include_monomers=include_monomers, ) path = None if isinstance(inputs, pd.DataFrame) else Path(inputs).resolve() if path is not None and path.name.lower().endswith( diff --git a/src/plinder/core/utils/config.py b/src/plinder/core/utils/config.py index ba8b3ce9d..3fd74f55e 100644 --- a/src/plinder/core/utils/config.py +++ b/src/plinder/core/utils/config.py @@ -15,6 +15,9 @@ LOG = setup_logger(__name__) +# Default public release: a YYYY-MM PDB snapshot stamp on the Cameo file server. +DEFAULT_RELEASE = "2026-09" + def _validate_cfg(*, cfg: DictConfig, schema: dict[str, Any]) -> DictConfig: """ @@ -220,14 +223,15 @@ class DataConfig: set automatically plinder_remote : str set automatically; the release root under ``PLINDER_MIRROR_URL`` - (default ``https://plinderdata.org``) + (default ``https://cameo3d.org/plinder``), in ``PLINDER-YYYY-MM`` + directories """ plinder_release: str = field( - default_factory=partial(_getenv_default, "PLINDER_RELEASE", "2026-07") + default_factory=partial(_getenv_default, "PLINDER_RELEASE", DEFAULT_RELEASE) ) plinder_release_number: str = field( - default_factory=partial(_getenv_default, "PLINDER_RELEASE_NUMBER", "1") + default_factory=partial(_getenv_default, "PLINDER_RELEASE_NUMBER", "") ) plinder_mount: str = field( default_factory=partial( @@ -254,8 +258,8 @@ def __post_init__(self) -> None: else: root = Path(self.plinder_mount) / self.plinder_bucket self.plinder_dir = (root / suffix).as_posix() if suffix else root.as_posix() - base = getenv("PLINDER_MIRROR_URL", "https://plinderdata.org").rstrip("/") - self.plinder_remote = f"{base}/{suffix}" if suffix else base + base = getenv("PLINDER_MIRROR_URL", "https://cameo3d.org/plinder").rstrip("/") + self.plinder_remote = f"{base}/PLINDER-{suffix}" if suffix else base @dataclass diff --git a/src/plinder/core/utils/cpl.py b/src/plinder/core/utils/cpl.py index dff926044..1b29a507a 100644 --- a/src/plinder/core/utils/cpl.py +++ b/src/plinder/core/utils/cpl.py @@ -1,6 +1,7 @@ -"""Public dataset access with cloudpathlib and a local immutable-release cache.""" +"""Public dataset access with cloudpathlib and a local release cache.""" import os +import re from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Callable, Iterable, TypeVar @@ -9,7 +10,7 @@ from tqdm import tqdm from plinder.core.utils.config import get_config -from plinder.core.utils.r2 import ReleaseClient, checked_key, manifest +from plinder.core.utils.http import ReleaseClient, checked_key T = TypeVar("T") @@ -39,12 +40,12 @@ def thread_pool(func: Callable[[T], None], items: Iterable[T]) -> None: def _get_client() -> ReleaseClient: cfg = get_config().data - if (cfg.plinder_bucket, cfg.plinder_release, cfg.plinder_release_number) != ( - "plinder", - "2024-06", - "v2", + if cfg.plinder_release_number or not re.fullmatch( + r"\d{4}-(0[1-9]|1[0-2])", cfg.plinder_release ): - raise ValueError("R2 downloads support only plinder/2024-06/v2") + raise ValueError( + "HTTP downloads support only YYYY-MM releases without a release number" + ) remote = str(cfg.plinder_remote).rstrip("/") url = urlsplit(remote) if ( @@ -55,9 +56,7 @@ def _get_client() -> ReleaseClient: or url.username or url.password ): - raise ValueError("PLINDER_MIRROR_URL must be an HTTP(S) bucket URL") - if cfg.force_update: - manifest.cache_clear() + raise ValueError("PLINDER_MIRROR_URL must be an HTTP(S) file-server URL") return ReleaseClient(remote) @@ -67,14 +66,18 @@ def _download(client: ReleaseClient, paths: list[Path], force_progress: bool) -> def fetch(path: Path) -> None: key = checked_key(path.resolve().relative_to(root).as_posix()) - if key not in client.records: - raise FileNotFoundError(f"File absent from release manifest: {key}") + remote = client.path(key) + if not remote.exists(): + raise FileNotFoundError(f"File absent from release: {key}") + metadata = client.metadata(remote) if ( cfg.force_update or not path.is_file() - or path.stat().st_size != client.records[key]["size"] + or path.stat().st_size != metadata.size + or metadata.modified is None + or path.stat().st_mtime != metadata.modified ): - client.path(key).download_to(path) + remote.download_to(path) if path.suffix == ".zip": path.with_name(path.stem + "_done").unlink(missing_ok=True) @@ -105,6 +108,20 @@ def download_paths(*, paths: list[Path], force_progress: bool = False) -> None: _download(_get_client(), paths, force_progress) +def _prune_unpublished(directory: Path, published: set[str]) -> None: + """Remove cached Parquet files that the release no longer lists. + + Directory artifacts are read as whole Parquet datasets, so a file dropped + from an updated release would otherwise be read alongside its replacement. + """ + root = Path(get_config().data.plinder_dir) + for path in directory.rglob("*.parquet"): + if path.is_symlink() or not path.is_file(): + continue + if path.relative_to(root).as_posix() not in published: + path.unlink() + + def get_plinder_path( *, rel: str = "", download: bool = True, force_progress: bool = False ) -> Path: @@ -132,12 +149,17 @@ def get_plinder_path( local = root / checked_key(rel) if is_offline() or not download: return local + local.resolve().relative_to(root.resolve()) client = _get_client() remote = client.path(rel) if not remote.exists(): - raise FileNotFoundError(f"Path absent from release manifest: {rel}") + raise FileNotFoundError(f"Path absent from release: {rel}") files = ( [remote] if remote.is_file() else [p for p in remote.rglob("*") if p.is_file()] ) - _download(client, [root / client.key(p) for p in files], force_progress) + keys = {client.key(p) for p in files} + _download(client, [root / key for key in sorted(keys)], force_progress) + if not remote.is_file(): + local.mkdir(parents=True, exist_ok=True) + _prune_unpublished(local, keys) return local diff --git a/src/plinder/core/utils/http.py b/src/plinder/core/utils/http.py new file mode 100644 index 000000000..4d3fa90a4 --- /dev/null +++ b/src/plinder/core/utils/http.py @@ -0,0 +1,232 @@ +"""Directory listing and atomic downloads from the public PLINDER file server.""" + +import os +import shutil +from dataclasses import dataclass +from email.utils import parsedate_to_datetime +from html.parser import HTMLParser +from http.client import HTTPException +from os import PathLike +from pathlib import Path +from tempfile import TemporaryDirectory +from time import sleep +from typing import Callable, Iterator, TypeVar, cast +from urllib.error import HTTPError +from urllib.parse import quote, unquote, urlsplit +from urllib.request import BaseHandler, Request + +from cloudpathlib import CloudPath as BaseCloudPath +from cloudpathlib import HttpPath, HttpsClient, HttpsPath + +CloudPathT = TypeVar("CloudPathT", bound=BaseCloudPath) +T = TypeVar("T") + + +def checked_key(key: str) -> str: + if key and ("\\" in key or any(p in {"", ".", ".."} for p in key.split("/"))): + raise ValueError(f"Invalid dataset path: {key!r}") + return key + + +class _TimeoutHandler(BaseHandler): + def http_request(self, request: Request) -> Request: + # Bound connection/read inactivity without limiting total transfer time. + request.timeout = 60 + return request + + https_request = http_request + + +def _retry(operation: Callable[[], T]) -> T: + for attempt in range(3): + try: + return operation() + except (OSError, HTTPException) as exc: + if isinstance(exc, HTTPError) and exc.code not in { + 408, + 429, + 500, + 502, + 503, + 504, + }: + raise + if attempt == 2: + raise + sleep(2**attempt) + raise AssertionError("unreachable") + + +class _DirectoryIndex(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.links: list[str] = [] + self.title = "" + self.in_title = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "title": + self.in_title = True + if tag == "a": + self.links.extend( + value for name, value in attrs if name == "href" and value + ) + + def handle_endtag(self, tag: str) -> None: + if tag == "title": + self.in_title = False + + def handle_data(self, data: str) -> None: + if self.in_title: + self.title += data + + +@dataclass(frozen=True) +class FileMetadata: + size: int + modified: float | None + is_dir: bool + + +class ReleaseClient(HttpsClient): + """Read a fresh directory index and HTTP metadata for each cache access.""" + + def __init__(self, remote: str): + super().__init__(auth=_TimeoutHandler()) + self.remote = remote.rstrip("/") + # These caches live only for one get_plinder_path/download_paths call. + self._metadata: dict[str, FileMetadata] = {} + self._listed: dict[str, bool] = {} + self.dir_matcher = self._is_directory + + def CloudPath(self, path: str | CloudPathT, *parts: str) -> CloudPathT: + cls = HttpsPath if self.remote.startswith("https:") else HttpPath + return cast(CloudPathT, cls(str(path), *parts, client=self)) + + def path(self, key: str = "") -> HttpPath: + return cast( + HttpPath, + self.CloudPath( + self.remote + ("/" + quote(checked_key(key), safe="/") if key else "") + ), + ) + + def key(self, path: str | BaseCloudPath) -> str: + url = str(path).removesuffix("/") + if url == self.remote: + return "" + if not url.startswith(self.remote + "/"): + raise ValueError("Path is outside the release") + return checked_key(unquote(url[len(self.remote) + 1 :])) + + def metadata(self, path: HttpPath) -> FileMetadata: + key = self.key(path) + if key not in self._metadata: + + def read() -> FileMetadata: + request = Request( + str(path), method="HEAD", headers={"Cache-Control": "no-cache"} + ) + with self.opener.open(request) as response: + if self.key(response.geturl()) != key: + raise ValueError( + "File server redirected outside the requested path" + ) + is_dir = response.geturl().endswith("/") + size = response.headers.get("Content-Length") + if not is_dir and size is None: + raise ValueError("File server must supply Content-Length") + modified = response.headers.get("Last-Modified") + return FileMetadata( + int(size or 0), + parsedate_to_datetime(modified).timestamp() + if modified + else None, + is_dir, + ) + + self._metadata[key] = _retry(read) + return self._metadata[key] + + def _is_directory(self, url: str) -> bool: + key = self.key(url) + if key in self._listed: + return self._listed[key] + return self.metadata(self.path(key)).is_dir + + def _exists(self, path: HttpPath) -> bool: + if self.key(path) in self._listed: + return True + try: + self.metadata(path) + except HTTPError as exc: + if exc.code == 404: + return False + raise + return True + + def _list_dir( + self, path: HttpPath, recursive: bool + ) -> Iterator[tuple[HttpPath, bool]]: + key = self.key(path) + url = str(self.path(key)) + "/" + + def read() -> str: + request = Request(url, headers={"Cache-Control": "no-cache"}) + with self.opener.open(request) as response: + if self.key(response.geturl()) != key: + raise ValueError( + "File server redirected outside the requested directory" + ) + return cast(bytes, response.read()).decode("utf-8") + + index = _DirectoryIndex() + index.feed(_retry(read)) + if not index.title.startswith(("Index of ", "Directory listing for ")): + raise ValueError("Expected an HTTP directory index") + children: dict[str, bool] = {} + for href in index.links: + link = urlsplit(href) + if link.scheme or link.netloc or link.query or link.fragment: + continue + name = unquote(link.path.removesuffix("/")) + # Only immediate relative children: ignore parent/sort/navigation links. + if not name or name in {".", ".."} or "/" in name or "\\" in name: + continue + child = checked_key(f"{key}/{name}" if key else name) + children[child] = link.path.endswith("/") + self._listed.update(children) + for child, is_dir in sorted(children.items()): + child_path = self.path(child) + yield child_path, is_dir + if recursive and is_dir: + yield from self._list_dir(child_path, recursive=True) + + def _download_file(self, path: HttpPath, local_path: str | PathLike[str]) -> Path: + destination = Path(local_path) + destination.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(dir=destination.parent) as tmp: + temporary = Path(tmp) / "download" + + def download() -> None: + request = Request(str(path), headers={"Cache-Control": "no-cache"}) + with self.opener.open(request) as response: + if self.key(response.geturl()) != self.key(path): + raise ValueError( + "File server redirected outside the requested path" + ) + size = response.headers.get("Content-Length") + if size is None: + raise ValueError("File server must supply Content-Length") + with temporary.open("wb") as stream: + shutil.copyfileobj(response, stream) + if temporary.stat().st_size != int(size): + raise OSError("Incomplete dataset download") + modified = response.headers.get("Last-Modified") + if modified: + timestamp = parsedate_to_datetime(modified).timestamp() + os.utime(temporary, (timestamp, timestamp)) + + _retry(download) + temporary.replace(destination) + return destination diff --git a/src/plinder/core/utils/r2.py b/src/plinder/core/utils/r2.py deleted file mode 100644 index 57c4be58a..000000000 --- a/src/plinder/core/utils/r2.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Manifest-backed listing and verified cloudpathlib downloads for public R2.""" - -import base64 -import gzip -import hashlib -import json -from functools import lru_cache -from http.client import HTTPException -from os import PathLike -from pathlib import Path -from tempfile import TemporaryDirectory -from time import sleep -from typing import Iterator, TypedDict, TypeVar, cast -from urllib.error import HTTPError -from urllib.parse import quote, unquote -from urllib.request import BaseHandler, Request - -from cloudpathlib import CloudPath as BaseCloudPath -from cloudpathlib import HttpPath, HttpsClient, HttpsPath - -CloudPathT = TypeVar("CloudPathT", bound=BaseCloudPath) - - -class ManifestRecord(TypedDict): - key: str - size: int - md5: str - - -def checked_key(key: str) -> str: - if key and ("\\" in key or any(p in {"", ".", ".."} for p in key.split("/"))): - raise ValueError(f"Invalid dataset path: {key!r}") - return key - - -class _TimeoutHandler(BaseHandler): - def http_request(self, request: Request) -> Request: - # Bound connection/read inactivity without limiting total transfer time. - request.timeout = 60 - return request - - https_request = http_request - - -class _DownloadClient(HttpsClient): - def __init__(self) -> None: - super().__init__(auth=_TimeoutHandler()) - - def _download( - self, - path: HttpPath, - local_path: str | PathLike[str], - expected_size: int | None = None, - ) -> Path: - destination = Path(local_path) - for attempt in range(3): - try: - super()._download_file(path, destination) - if ( - expected_size is not None - and destination.stat().st_size != expected_size - ): - raise OSError("Incomplete dataset download") - break - except (OSError, HTTPException) as exc: - if isinstance(exc, HTTPError) and exc.code not in { - 408, - 429, - 500, - 502, - 503, - 504, - }: - raise - if attempt == 2: - raise - sleep(2**attempt) - return destination - - -@lru_cache(maxsize=4) -def manifest(remote: str) -> dict[str, ManifestRecord]: - with TemporaryDirectory() as tmp: - path = Path(tmp) / "manifest.jsonl.gz" - path_class = HttpsPath if remote.startswith("https:") else HttpPath - client = _DownloadClient() - source = path_class(remote + "/manifest.jsonl.gz", client=client) - # A direct GET preserves transient errors hidden by cloudpathlib's HEAD check. - client._download(source, path) - data = path.read_bytes() - records: dict[str, ManifestRecord] = {} - for line in gzip.decompress(data).splitlines(): - row = json.loads(line) - key = checked_key(row["key"]) - if not key or key in records or type(row["size"]) is not int or row["size"] < 0: - raise ValueError("Invalid or duplicate manifest record") - if len(base64.b64decode(row["md5"], validate=True)) != 16: - raise ValueError("Invalid manifest MD5") - records[key] = row - return records - - -class ReleaseClient(_DownloadClient): - """Use the manifest where public HTTP lacks object listing; delegate transfers.""" - - def __init__(self, remote: str): - super().__init__() - self.remote = remote - self.records = manifest(remote) - self.dir_matcher = lambda url: self.key(url) not in self.records - - def CloudPath(self, path: str | CloudPathT, *parts: str) -> CloudPathT: - cls = HttpsPath if self.remote.startswith("https:") else HttpPath - return cast(CloudPathT, cls(str(path), *parts, client=self)) - - def path(self, key: str = "") -> HttpPath: - return cast( - HttpPath, - self.CloudPath( - self.remote + ("/" + quote(checked_key(key), safe="/") if key else "") - ), - ) - - def key(self, path: str | BaseCloudPath) -> str: - url = str(path) - if url == self.remote: - return "" - if not url.startswith(self.remote + "/"): - raise ValueError("Path is outside the release") - return checked_key(unquote(url[len(self.remote) + 1 :])) - - def _exists(self, path: HttpPath) -> bool: - key = self.key(path) - return key in self.records or any( - not key or k.startswith(key + "/") for k in self.records - ) - - def _list_dir( - self, path: HttpPath, recursive: bool - ) -> Iterator[tuple[HttpPath, bool]]: - key = self.key(path) - prefix = key + "/" if key else "" - children = {} - for name in self.records: - if name.startswith(prefix): - parts = name[len(prefix) :].split("/") - for i in range(1, len(parts) + 1 if recursive else 2): - children[prefix + "/".join(parts[:i])] = i < len(parts) - for name, is_dir in sorted(children.items()): - yield self.path(name), is_dir - - def _download_file(self, path: HttpPath, local_path: str | PathLike[str]) -> Path: - destination = Path(local_path) - destination.parent.mkdir(parents=True, exist_ok=True) - record = self.records[self.key(path)] - with TemporaryDirectory(dir=destination.parent) as tmp: - temporary = Path(tmp) / "download" - self._download(path, temporary, expected_size=record["size"]) - digest = hashlib.md5() - with temporary.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - if base64.b64encode(digest.digest()).decode() != record["md5"]: - raise ValueError("Dataset checksum mismatch") - temporary.replace(destination) - return destination diff --git a/src/plinder/data/annotations/cif_utils.py b/src/plinder/data/annotations/cif_utils.py index d6efe789e..9730a8995 100644 --- a/src/plinder/data/annotations/cif_utils.py +++ b/src/plinder/data/annotations/cif_utils.py @@ -766,13 +766,17 @@ def drop_self_clashing_symmetry_copies( clash_ratio: float = 0.5, context: str = "assembly", ) -> struc.AtomArray: - """Drop symmetry copies overlapping a lower ``sym_id`` copy. + """Drop coincident symmetry copies and severely overlapping protein chains. Chains on a symmetry axis can expand into coincident copies of one ``label_asym_id``. Copies are visited in ascending ``sym_id``; one is dropped when more than ``clash_ratio`` of its atoms lie within ``clash_distance`` of an already kept copy. Waters are untouched. - Requires the ``sym_id`` and ``label_asym_id`` annotations. + Protein chains from different source asym IDs are also compared by their + C-alpha atoms: a chain is removed if at least five and more than 10%, or + at least twenty regardless of chain length, lie within 2 Γ… of previously + retained protein chains. Such C-alpha distances are nonphysical for + separate chains. Requires the ``sym_id`` and ``label_asym_id`` annotations. """ categories = set(assembly.get_annotation_categories()) if not {"sym_id", "label_asym_id"}.issubset(categories): @@ -815,6 +819,65 @@ def drop_self_clashing_symmetry_copies( asym_id, [(sym_id, round(fraction, 2)) for sym_id, fraction in dropped], ) + ca_indices = np.flatnonzero( + keep + & candidates + & (assembly.atom_name == "CA") + & struc.filter_amino_acids(assembly) + ) + if len(ca_indices): + ca = assembly[ca_indices] + chain_names = np.array( + [ + f"{int(sym_id)}.{asym_id}" + for sym_id, asym_id in zip(ca.sym_id, ca.label_asym_id, strict=True) + ] + ) + unique_chains, chain_index = np.unique(chain_names, return_inverse=True) + chain_sizes = np.bincount(chain_index) + eligible = [index for index, size in enumerate(chain_sizes) if size >= 10] + if len(eligible) > 1: + cell_list = struc.CellList(ca, cell_size=3.0) + retained: list[int] = [] + dropped_chains: list[tuple[str, int, float]] = [] + for index in sorted( + eligible, + key=lambda value: ( + int(unique_chains[value].split(".", 1)[0]), + unique_chains[value], + ), + ): + query_mask = chain_index == index + if retained: + neighbors = cell_list.get_atoms(ca.coord[query_mask], radius=2.0) + clashing = ( + (neighbors >= 0) + & np.isin(chain_index[np.clip(neighbors, 0, None)], retained) + ).any(axis=1) + fraction = float(clashing.mean()) + clash_count = int(clashing.sum()) + if clash_count >= 20 or (clash_count >= 5 and fraction > 0.1): + sym_id, asym_id = unique_chains[index].split(".", 1) + keep &= ~( + (assembly.sym_id == int(sym_id)) + & (assembly.label_asym_id == asym_id) + ) + dropped_chains.append( + (str(unique_chains[index]), clash_count, fraction) + ) + continue + retained.append(index) + if dropped_chains: + LOG.info( + "%s: dropped %d clashing protein chains (first 10 chain, " + "C-alpha clash count, fraction): %s", + context, + len(dropped_chains), + [ + (chain, count, round(fraction, 2)) + for chain, count, fraction in dropped_chains[:10] + ], + ) if keep.all(): return assembly return assembly[keep] diff --git a/src/plinder/data/annotations/get_similarity_scores.py b/src/plinder/data/annotations/get_similarity_scores.py index 49e07050e..e0487d8e5 100644 --- a/src/plinder/data/annotations/get_similarity_scores.py +++ b/src/plinder/data/annotations/get_similarity_scores.py @@ -2751,9 +2751,13 @@ def load_alignments( aln_df = aln_df.drop(columns="_merge") for column in score_columns: aln_df[column] = aln_df[column].astype(float) / 100 + # Custom-scoring alignments keep their CIGARs but are already + # mapped to residue numbers, so only undecoded release shards + # need the chain lookup. if ( "cigar" in aln_df.columns and "query_selected_residue_positions" not in aln_df.columns + and "query_selected_residue_numbers" not in aln_df.columns ): if self.alignment_chain_lookup is None: raise ValueError("CIGAR alignments require a chain lookup") diff --git a/src/plinder/data/clusters.py b/src/plinder/data/clusters.py index d138f8368..9bfadb7d5 100644 --- a/src/plinder/data/clusters.py +++ b/src/plinder/data/clusters.py @@ -23,7 +23,6 @@ import networkit as nk import numpy as np import pandas as pd -import pyarrow as pa import pyarrow.parquet as pq from numpy.typing import NDArray @@ -49,6 +48,12 @@ INTERFACE_MEMBERSHIP = Path("index/interface_membership.parquet") +def _set_slurm_memory_limit(connection: Any, share: int) -> None: + memory_mb = int(os.environ.get("SLURM_MEM_PER_NODE", "0")) + if memory_mb > 0: + connection.execute("SET memory_limit = ?", [f"{memory_mb * share // 5}MB"]) + + def _cluster_root(data_dir: Path, entity_type: ClusterEntity) -> Path: if entity_type == "ligand": return data_dir / "ligand_clusters" @@ -583,13 +588,27 @@ def prepare_symmetric_edge_plan( score_metrics = [ metric for metric in selected_metrics if not is_chemical_cluster_metric(metric) ] - # One aggregation group per distinct raw-source set: interface metrics share - # the interface score shards, all protein/pocket metrics share the holo - # search scores, and each chemical metric has its own directed-edge shards - # (e.g. ligand_scores / mhfp6_scores). + # Group metrics by score source; interface exports and chemical metrics + # have separate sources, while protein/pocket metrics share holo scores. source_groups: list[tuple[str, list[str]]] = [] if entity_type == "interface": - source_groups.append(("interface", selected_metrics)) + raw_metrics: list[str] = [] + for metric in selected_metrics: + export = ( + data_dir + / "exports" + / ( + "interface_similarity_scores.parquet" + if metric == "interface_qcov" + else "interface_half_similarity_scores.parquet" + ) + ) + if export.is_file(): + source_groups.append((f"interface_export_{metric}", [metric])) + else: + raw_metrics.append(metric) + if raw_metrics: + source_groups.append(("interface", raw_metrics)) else: if score_metrics: source_groups.append(("score", score_metrics)) @@ -598,11 +617,19 @@ def prepare_symmetric_edge_plan( for kind, source_metrics in source_groups: if not source_metrics: continue - sources = _raw_component_score_sources( - data_dir=data_dir, - metric=source_metrics[0], - entity_type=entity_type, - ) + if kind.startswith("interface_export_"): + export_name = ( + "interface_similarity_scores.parquet" + if source_metrics[0] == "interface_qcov" + else "interface_half_similarity_scores.parquet" + ) + sources = [data_dir / "exports" / export_name] + else: + sources = _raw_component_score_sources( + data_dir=data_dir, + metric=source_metrics[0], + entity_type=entity_type, + ) if not sources: if entity_type != "interface": raise FileNotFoundError( @@ -792,7 +819,7 @@ def write_symmetric_edge_fragment_batch( read_paths: Sequence[Path] | None = None, entity_type: ClusterEntity = "ligand", ) -> dict[str, Any]: - """Aggregate one raw-source batch into canonical-pair hash fragments.""" + """Write one score batch into canonical-pair hash fragments.""" if threads < 1: raise ValueError("symmetric-edge threads must be positive") plan = load_symmetric_edge_plan(data_dir, entity_type=entity_type) @@ -825,6 +852,7 @@ def write_symmetric_edge_fragment_batch( import duckdb connection = duckdb.connect() + _set_slurm_memory_limit(connection, 3) connection.sql(f"SET threads={threads}") temporary_root = scratch_dir / "duckdb" temporary_root.mkdir(exist_ok=True, parents=True) @@ -861,6 +889,26 @@ def write_symmetric_edge_fragment_batch( WHERE cast(metric AS VARCHAR) IN ({metrics_sql}) """ ) + elif batch["kind"] in { + "interface_export_interface_qcov", + "interface_export_interface_side_qcov", + }: + metric = metrics[0] + query, target = ( + ("query_system", "target_system") + if metric == "interface_qcov" + else ("query_half_interface_id", "target_half_interface_id") + ) + selected_sql = dedent( + f""" + SELECT + '{metric}'::VARCHAR AS metric, + cast({query} AS VARCHAR) AS query_node, + cast({target} AS VARCHAR) AS target_node, + cast(similarity AS DOUBLE) AS similarity + FROM read_parquet([{paths_sql}], union_by_name=true) + """ + ) else: nodes, eligible_systems = component_node_universe( data_dir=data_dir, @@ -930,6 +978,39 @@ def write_symmetric_edge_fragment_batch( max(similarity) FILTER (WHERE query_node = high_node)::DOUBLE AS reverse_similarity """ + compact_interface_export = batch["kind"] in { + "interface_export_interface_qcov", + "interface_export_interface_side_qcov", + } + edge_rows_sql = ( + dedent( + """ + SELECT + metric, + bucket, + low_node, + high_node, + CASE WHEN query_node = low_node THEN similarity END + AS forward_similarity, + CASE WHEN query_node = high_node THEN similarity END + AS reverse_similarity + FROM bucketed + """ + ) + if compact_interface_export + else dedent( + f""" + SELECT + metric, + bucket, + low_node, + high_node, + {aggregate_sql} + FROM bucketed + GROUP BY metric, bucket, low_node, high_node + """ + ) + ) query = dedent( f""" COPY ( @@ -955,14 +1036,7 @@ def write_symmetric_edge_fragment_batch( )::INTEGER AS bucket FROM canonical ) - SELECT - metric, - bucket, - low_node, - high_node, - {aggregate_sql} - FROM bucketed - GROUP BY metric, bucket, low_node, high_node + {edge_rows_sql} ) TO '{scratch_output.as_posix()}' ( FORMAT PARQUET, @@ -1102,6 +1176,7 @@ def write_symmetric_edge_shard( local_output = scratch_dir / f"{metric}-{bucket:03d}.parquet" local_output.unlink(missing_ok=True) connection = duckdb.connect() + _set_slurm_memory_limit(connection, 3) connection.sql(f"SET threads={threads}") connection.sql(f"SET temp_directory='{scratch_dir.as_posix()}'") if fragments: @@ -2983,11 +3058,12 @@ def _stage_directed_cover_edges( import duckdb connection = duckdb.connect() + _set_slurm_memory_limit(connection, 4) connection.sql(f"SET threads={threads}") + connection.sql("SET preserve_insertion_order=false") temporary_root.mkdir(exist_ok=True, parents=True) escaped_temporary_root = temporary_root.as_posix().replace("'", "''") connection.sql(f"SET temp_directory='{escaped_temporary_root}'") - connection.sql("SET preserve_insertion_order=false") connection.register( "component_labels", labels[[node_column, "graph_node"]], @@ -3041,35 +3117,30 @@ def _stage_directed_cover_edges( ) staged_edges.unlink(missing_ok=True) incoming_counts_path.unlink(missing_ok=True) - staged_schema = pa.schema( - [ - ("query_node", pa.uint32()), - ("target_node", pa.uint32()), - ("similarity", pa.uint8()), - ] - ) staged_rows = 0 incoming_counts = np.zeros(len(labels), dtype=np.int64) stage_started = time() - writer = pq.ParquetWriter( - staged_edges, - staged_schema, - compression="zstd", - use_dictionary=False, - write_statistics=False, - ) try: - reader = connection.execute(query).fetch_record_batch(rows_per_batch=2_000_000) - for batch_index, record_batch in enumerate(reader, start=1): - compact = ( - pa.Table.from_batches([record_batch]) - .select(staged_schema.names) - .cast(staged_schema, safe=False) - ) - writer.write_table(compact, row_group_size=2_000_000) - staged_rows += len(record_batch) - target_index = compact.schema.get_field_index("target_node") - targets = compact.column(target_index).to_numpy(zero_copy_only=False) + escaped_staged_edges = staged_edges.as_posix().replace("'", "''") + connection.execute( + f"COPY ({query}) TO '{escaped_staged_edges}' " + "(FORMAT PARQUET, COMPRESSION ZSTD, ROW_GROUP_SIZE 2000000)" + ) + previous_target = -1 + for batch_index, batch in enumerate( + pq.ParquetFile(staged_edges).iter_batches( + batch_size=2_000_000, columns=["target_node"] + ), + start=1, + ): + targets = batch.column(0).to_numpy(zero_copy_only=False) + if len(targets) and ( + int(targets[0]) < previous_target or np.any(targets[1:] < targets[:-1]) + ): + raise ValueError("directed-cover edges are not target-sorted") + if len(targets): + previous_target = int(targets[-1]) + staged_rows += len(targets) incoming_counts += np.bincount(targets, minlength=len(labels)) if batch_index % 5 == 0: elapsed = time() - stage_started @@ -3084,13 +3155,11 @@ def _stage_directed_cover_edges( elapsed, ) except BaseException: - writer.close() connection.close() staged_edges.unlink(missing_ok=True) incoming_counts_path.unlink(missing_ok=True) raise else: - writer.close() connection.close() np.save(incoming_counts_path, incoming_counts, allow_pickle=False) LOG.info( diff --git a/src/plinder/data/databases.py b/src/plinder/data/databases.py index 88e18befd..16846291f 100644 --- a/src/plinder/data/databases.py +++ b/src/plinder/data/databases.py @@ -280,7 +280,7 @@ def _search_database_bundle_sources( f"unsafe database prefix in {source_root / 'exact_cluster.json'}" ) prefixes = {value for value in values if isinstance(value, str)} - sources = {source_root / "exact_cluster.json"} + sources: set[Path] = set() for prefix in prefixes: sources.update( path @@ -812,6 +812,26 @@ def make_sub_dbs( ) ) working_database = working_subdb / working_subdb.name + if search_db_aln_type.startswith("monomer_") and not ( + working_database.with_suffix(".index").is_file() + and _database_entry_count(working_database) + ): + # linclust crashes on an empty database. An empty monomer + # universe is not an error: downstream bundling skips it. + # Retire any earlier build so it is not published as current. + LOG.info(f"no {search_db_aln_type} chains selected; skipping") + for stale in ("exact_cluster.json", "selection.json"): + (subdb / stale).unlink(missing_ok=True) + for prefix in ( + subdb.name, + "exact_clusters", + "representatives", + "cluster_alignments", + "clustered", + ): + _remove_database_prefix(subdb / prefix) + report[search_db_aln_type] = missing + continue cluster_manifest = make_exact_search_db( full_db=working_database, aln_type=aln_type, diff --git a/src/plinder/data/monomer_similarity.py b/src/plinder/data/monomer_similarity.py new file mode 100644 index 000000000..a76ecc7d3 --- /dev/null +++ b/src/plinder/data/monomer_similarity.py @@ -0,0 +1,321 @@ +"""Precompute chain scores for proteins outside ligand pockets and interfaces.""" + +from __future__ import annotations + +import argparse +import shutil +from collections.abc import Iterator +from functools import lru_cache +from pathlib import Path + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.parquet as pq + +from plinder.core.scores.custom import resolve_search_database +from plinder.core.utils.schemas import PROTEIN_SIMILARITY_EXPORT_SCHEMA +from plinder.data import databases +from plinder.data.annotations.get_similarity_scores import ( + get_sequence_similarity_helper, + run_alignment, +) +from plinder.data.pipeline.config import FoldseekConfig, MMSeqsConfig + + +def _chain_ids(data_dir: Path, backend: str) -> dict[str, tuple[str, str]]: + chains = pd.read_parquet( + data_dir / "index/alignment_chain_lookup.parquet", + columns=["entry_pdb_id", "chain_auth_id", "chain_asym_id"], + ) + prefix = "pdb_0000" if backend == "foldseek" else "" + infix = "_xyz-enrich_" if backend == "foldseek" else "_" + identifiers = ( + prefix + + chains["entry_pdb_id"].astype(str) + + infix + + chains["chain_auth_id"].astype(str) + ) + if identifiers.duplicated().any(): + raise ValueError("release chain lookup has ambiguous author-chain IDs") + return dict( + zip( + identifiers, + zip(chains["entry_pdb_id"], chains["chain_asym_id"], strict=True), + strict=True, + ) + ) + + +def _percent(values: pd.Series, *, allow_missing: bool = False) -> pa.Array: + numeric = pd.to_numeric(values, errors="raise").to_numpy(dtype=float) + missing = np.isnan(numeric) + if (not allow_missing and missing.any()) or np.isinf(numeric).any(): + raise ValueError("protein alignment contains a non-finite score") + scaled = np.floor(np.nan_to_num(numeric, nan=0) * 100 + 0.5) + return pa.array(scaled.clip(0, 100).astype("uint8"), mask=missing) + + +def _alignment_frames(scanner: ds.Scanner) -> Iterator[pd.DataFrame]: + """Combine small search row groups before converting and writing scores.""" + pending: list[pa.RecordBatch] = [] + count = 0 + for batch in scanner.to_batches(): + if not batch.num_rows: + continue + pending.append(batch) + count += batch.num_rows + if count >= 100_000: + yield pa.Table.from_batches(pending).to_pandas() + pending.clear() + count = 0 + if pending: + yield pa.Table.from_batches(pending).to_pandas() + + +def _score_alignment_batches( + raw_path: Path, + output_path: Path, + *, + backend: str, + chain_ids: dict[str, tuple[str, str]], +) -> int: + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary = output_path.with_suffix(".tmp.parquet") + count = 0 + + @lru_cache(maxsize=100_000) + def sequence_similarity(query: str, target: str) -> float: + return get_sequence_similarity_helper(query, target) + + with pq.ParquetWriter( + temporary, PROTEIN_SIMILARITY_EXPORT_SCHEMA, compression="zstd" + ) as writer: + scanner = ds.dataset(raw_path, format="parquet").scanner( + columns=[ + "query", + "target", + "qcov", + "tcov", + "fident", + "qaln", + "taln", + *(["lddt"] if backend == "foldseek" else []), + ], + batch_size=100_000, + ) + for rows in _alignment_frames(scanner): + query_ids = rows["query"] + target_ids = rows["target"] + if backend == "foldseek": + model_prefix = r"_xyz-enrich_MODEL_\d+_" + query_ids = query_ids.str.replace( + model_prefix, "_xyz-enrich_", regex=True + ) + target_ids = target_ids.str.replace( + model_prefix, "_xyz-enrich_", regex=True + ) + queries = query_ids.map(chain_ids) + targets = target_ids.map(chain_ids) + if queries.isna().any() or targets.isna().any(): + unknown = { + "query": rows.loc[queries.isna(), "query"].head(5).tolist(), + "target": rows.loc[targets.isna(), "target"].head(5).tolist(), + } + raise ValueError(f"monomer alignment has unknown chains: {unknown}") + query_entry, query_chain = zip(*queries, strict=True) + target_entry, target_chain = zip(*targets, strict=True) + similarity = pd.Series( + [ + sequence_similarity(str(q).upper(), str(t).upper()) + for q, t in zip(rows["qaln"], rows["taln"], strict=True) + ] + ) + size = len(rows) + table = pa.Table.from_arrays( + [ + pa.array(query_entry, type=pa.string()), + pa.array(target_entry, type=pa.string()), + pa.array(query_chain, type=pa.string()), + pa.array(target_chain, type=pa.string()), + pa.array([backend] * size, type=pa.string()), + _percent(rows["qcov"]), + _percent(rows["tcov"]), + _percent(rows["fident"]), + _percent(similarity), + ( + _percent(rows["lddt"], allow_missing=True) + if backend == "foldseek" + else pa.nulls(size, type=pa.uint8()) + ), + ], + schema=PROTEIN_SIMILARITY_EXPORT_SCHEMA, + ) + writer.write_table(table) + count += size + temporary.replace(output_path) + return count + + +def score_monomer_pairs( + data_dir: Path, + *, + backend: str, + target_kind: str, + scratch_dir: Path, + output_path: Path, + threads: int, + pilot_queries: int | None = None, + query_batch_index: int | None = None, + query_batch_count: int | None = None, +) -> int: + """Score unrepresented protein query chains against one target universe.""" + if backend not in {"foldseek", "mmseqs"}: + raise ValueError(f"unsupported backend: {backend}") + if target_kind not in {"holo", "monomer"}: + raise ValueError(f"unsupported target universe: {target_kind}") + if threads < 1: + raise ValueError("threads must be positive") + data_dir = Path(data_dir) + scratch_dir = Path(scratch_dir) + scratch_dir.mkdir(parents=True, exist_ok=True) + query = data_dir / "dbs/subdbs" / f"monomer_{backend}" / f"monomer_{backend}" + if pilot_queries is not None or query_batch_index is not None: + identifiers = sorted(databases.database_identifiers(query)) + if pilot_queries is not None: + if pilot_queries < 1 or query_batch_index is not None: + raise ValueError("pilot_queries must be positive and unbatched") + identifiers = identifiers[:pilot_queries] + else: + if ( + query_batch_index is None + or query_batch_count is None + or not 0 <= query_batch_index < query_batch_count + ): + raise ValueError("invalid monomer query batch") + start = len(identifiers) * query_batch_index // query_batch_count + stop = len(identifiers) * (query_batch_index + 1) // query_batch_count + identifiers = identifiers[start:stop] + print(f"Searching {len(identifiers):,} monomer query chains", flush=True) + subset_root = scratch_dir / "query_subset" + subset_root.mkdir(exist_ok=True) + missing = databases.make_sub_db(set(identifiers), query, subset_root, backend) + if missing: + raise ValueError(f"missing pilot query chains: {sorted(missing)[:10]}") + query = subset_root / subset_root.name + + target = resolve_search_database( + backend, data_dir=data_dir, monomer=target_kind == "monomer" + ) + config = FoldseekConfig() if backend == "foldseek" else MMSeqsConfig() + raw = scratch_dir / "raw_alignment" + run_alignment( + aln_type=backend, + query_db=query, + target_db=target.conversion_target, + search_target_db=target.search_target, + cluster_alignment_db=target.cluster_alignments, + search_db=scratch_dir / "results", + aln_file=raw, + alignment_config=config, + tmp_dir=scratch_dir / "search_tmp", + threads=threads, + ) + try: + return _score_alignment_batches( + raw.with_suffix(".parquet"), + Path(output_path), + backend=backend, + chain_ids=_chain_ids(data_dir, backend), + ) + finally: + shutil.rmtree(raw.with_suffix(".parquet"), ignore_errors=True) + + +def finalize_monomer_similarity_scores( + data_dir: Path, *, query_batch_count: int = 8 +) -> int: + """Install the complete monomer-pair table after every batch succeeds.""" + if query_batch_count < 1: + raise ValueError("query_batch_count must be positive") + exports = Path(data_dir) / "exports" + staging = exports / ".monomer_similarity_scores_staging" + installing = exports / ".monomer_similarity_scores_installing" + installed = exports / "monomer_similarity_scores" + if installed.exists(): + raise FileExistsError(installed) + expected = { + staging + / f"alignment_type={backend}" + / f"target_kind={target_kind}" + / f"part-{batch_index}.parquet" + for backend in ("mmseqs", "foldseek") + for target_kind in ("holo", "monomer") + for batch_index in range(query_batch_count) + } + extra = set(staging.rglob("*.parquet")) - expected + if extra: + raise ValueError(f"unexpected monomer score batches: {sorted(extra)[:5]}") + total_rows = 0 + for backend in ("mmseqs", "foldseek"): + for target_kind in ("holo", "monomer"): + for batch_index in range(query_batch_count): + path = ( + staging + / f"alignment_type={backend}" + / f"target_kind={target_kind}" + / f"part-{batch_index}.parquet" + ) + if not path.is_file(): + raise FileNotFoundError(path) + file = pq.ParquetFile(path) + if not file.schema_arrow.equals(PROTEIN_SIMILARITY_EXPORT_SCHEMA): + raise ValueError(f"invalid monomer score schema: {path}") + total_rows += file.metadata.num_rows + if installing.exists(): + shutil.rmtree(installing) + for backend in ("mmseqs", "foldseek"): + backend_root = installing / f"alignment_type={backend}" + backend_root.mkdir(parents=True) + for target_kind in ("holo", "monomer"): + target_root = ( + staging / f"alignment_type={backend}" / f"target_kind={target_kind}" + ) + for batch_index in range(query_batch_count): + ( + backend_root / f"monomer_{target_kind}_{batch_index}.parquet" + ).hardlink_to(target_root / f"part-{batch_index}.parquet") + installing.rename(installed) + shutil.rmtree(staging, ignore_errors=True) + return total_rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("data_dir", type=Path) + parser.add_argument("--backend", choices=("foldseek", "mmseqs"), required=True) + parser.add_argument("--target-kind", choices=("holo", "monomer"), required=True) + parser.add_argument("--scratch-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--threads", type=int, default=16) + parser.add_argument("--pilot-queries", type=int) + parser.add_argument("--query-batch-index", type=int) + parser.add_argument("--query-batch-count", type=int) + args = parser.parse_args() + count = score_monomer_pairs( + args.data_dir, + backend=args.backend, + target_kind=args.target_kind, + scratch_dir=args.scratch_dir, + output_path=args.output, + threads=args.threads, + pilot_queries=args.pilot_queries, + query_batch_index=args.query_batch_index, + query_batch_count=args.query_batch_count, + ) + print(f"Wrote {count:,} {args.backend}/{args.target_kind} pair rows", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/plinder/data/pipeline/pipeline.py b/src/plinder/data/pipeline/pipeline.py index fc8142a69..bfa509d4c 100644 --- a/src/plinder/data/pipeline/pipeline.py +++ b/src/plinder/data/pipeline/pipeline.py @@ -342,9 +342,12 @@ def finalize_ligand_archives(self) -> None: @utils.ingest_flow_control def make_sub_dbs(self) -> None: + search_databases = list(self.cfg.scorer.sub_databases) + if "holo" in search_databases: + search_databases.append("monomer") tasks.make_sub_dbs( data_dir=self.plinder_dir, - sub_databases=self.cfg.scorer.sub_databases, + sub_databases=search_databases, cpu=self.cfg.flow.make_sub_dbs_cpu, scratch_dir=Path(tempfile.gettempdir()) / "plinder-exact-search-dbs", ) diff --git a/src/plinder/data/pipeline/public_release.py b/src/plinder/data/pipeline/public_release.py new file mode 100644 index 000000000..efcb05f64 --- /dev/null +++ b/src/plinder/data/pipeline/public_release.py @@ -0,0 +1,166 @@ +# Copyright (c) 2024, Plinder Development Team +# Distributed under the terms of the Apache License 2.0 +"""Build a clean public release tree from a working ingest.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import tempfile +from pathlib import Path + +from plinder.core.release import RELEASE_PATHS, RELEASE_TABLES + +_FILE_ARTIFACTS = ( + *(table["artifact"] for table in RELEASE_TABLES.values()), + "ligand_similarity_scores", + "interface_similarity_scores", + "interface_half_similarity_scores", +) +_PARQUET_DIRECTORIES = ( + "protein_similarity_scores", + "monomer_similarity_scores", + "ligand_archives", + "ligand_scores", + "alignment_cigars", +) +_SAMPLING_DIRECTORIES = ("ligand_sampling", "interface_sampling") +_SEARCH_DATABASES = ( + "holo_mmseqs", + "holo_foldseek", + "monomer_mmseqs", + "monomer_foldseek", +) +# Monomer artifacts exist only when the ingest had monomer chains to search. +_OPTIONAL = {"monomer_similarity_scores", "monomer_mmseqs", "monomer_foldseek"} + + +def _public_files(source: Path) -> list[Path]: + files = {Path(RELEASE_PATHS[name]) for name in _FILE_ARTIFACTS} + for name in _PARQUET_DIRECTORIES: + directory = source / RELEASE_PATHS.get(name, name) + if not directory.is_dir(): + if name in _OPTIONAL: + continue + raise FileNotFoundError(directory) + files.update(path.relative_to(source) for path in directory.rglob("*.parquet")) + for name in _SAMPLING_DIRECTORIES: + directory = source / RELEASE_PATHS.get(name, name) + if not directory.is_dir(): + raise FileNotFoundError(directory) + for cover in ("set_cover", "directed_set_cover"): + files.update( + path.relative_to(source) + for path in (directory / cover).glob("metric=*/threshold=*.parquet") + ) + + database_root = source / RELEASE_PATHS["search_databases"] + for name in _SEARCH_DATABASES: + directory = database_root / name + if not directory.is_dir(): + if name in _OPTIONAL: + continue + raise FileNotFoundError(directory) + files.update( + path.relative_to(source) + for path in directory.rglob("*") + if path.is_file() and path.suffix != ".json" + ) + shadowed = database_root / "shadowed_entries.parquet" + if shadowed.is_file(): + files.add(shadowed.relative_to(source)) + overlay = database_root / "weekly_delta" + if overlay.is_dir(): + files.update( + path.relative_to(source) + for path in overlay.rglob("*") + if path.is_file() and path.suffix != ".json" + ) + + missing = [str(source / path) for path in files if not (source / path).is_file()] + if missing: + raise FileNotFoundError(f"missing public release files: {missing[:10]}") + included = {source / item for item in files} + for relative in files: + path = source / relative + if path.is_symlink(): + if Path(os.readlink(path)).is_absolute(): + raise ValueError(f"absolute link in public release: {path}") + if path.resolve() not in included: + raise ValueError( + f"search database link leaves the public release: {path}" + ) + return sorted(files) + + +def prepare_public_release( + source: Path, destination: Path, *, replace: bool = False, copy: bool = False +) -> int: + """Stage only public files beside an ingest; return the number of files.""" + source = source.resolve(strict=True) + if destination.is_symlink(): + raise ValueError("public release destination cannot be a symlink") + destination = destination.parent.resolve() / destination.name + if ( + source == destination + or source in destination.parents + or destination in source.parents + ): + raise ValueError("working and public release directories must be separate") + if destination.exists() and not replace: + raise FileExistsError(destination) + if destination.exists() and not destination.is_dir(): + raise NotADirectoryError(destination) + files = _public_files(source) + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path( + tempfile.mkdtemp(prefix=f".{destination.name}-", dir=destination.parent) + ) + previous = staging.with_name(staging.name + "-previous") + try: + for relative in files: + original = source / relative + target = staging / relative + target.parent.mkdir(parents=True, exist_ok=True) + if original.is_symlink(): + target.symlink_to(os.readlink(original)) + elif copy: + shutil.copy2(original, target) + else: + os.link(original, target) + if destination.exists(): + destination.rename(previous) + try: + staging.rename(destination) + except OSError: + if previous.exists(): + previous.rename(destination) + raise + if previous.exists(): + shutil.rmtree(previous) + finally: + if staging.exists(): + shutil.rmtree(staging) + return len(files) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("working", type=Path, help="completed working ingest") + parser.add_argument("public", type=Path, help="separate public release directory") + parser.add_argument( + "--replace", action="store_true", help="replace an existing public tree" + ) + parser.add_argument( + "--copy", action="store_true", help="copy bytes instead of hard-linking" + ) + args = parser.parse_args() + count = prepare_public_release( + args.working, args.public, replace=args.replace, copy=args.copy + ) + print(f"Prepared {count} public files in {args.public}") + + +if __name__ == "__main__": + main() diff --git a/src/plinder/data/pipeline/score.py b/src/plinder/data/pipeline/score.py index 8d19e73a4..8d272df36 100644 --- a/src/plinder/data/pipeline/score.py +++ b/src/plinder/data/pipeline/score.py @@ -3417,7 +3417,6 @@ def publish_search_database_bundles( rmtree(path) staging.mkdir(parents=True) reports: dict[str, dict[str, int | str]] = {} - overlay_reports: dict[str, dict[str, int | str]] = {} try: for alignment_type in alignment_types: reports[alignment_type] = databases.publish_search_database_bundle( @@ -3425,31 +3424,25 @@ def publish_search_database_bundles( target_root=staging / f"holo_{alignment_type}", aln_type=alignment_type, ) + monomer_source = data_dir / "dbs" / "subdbs" / f"monomer_{alignment_type}" + if (monomer_source / "exact_cluster.json").is_file(): + databases.publish_search_database_bundle( + source_root=monomer_source, + target_root=staging / f"monomer_{alignment_type}", + aln_type=alignment_type, + ) overlay_source = ( data_dir / "dbs/weekly_delta/subdbs" / f"holo_{alignment_type}" ) if (overlay_source / "exact_cluster.json").is_file(): - overlay_reports[alignment_type] = ( - databases.publish_search_database_bundle( - source_root=overlay_source, - target_root=staging / "weekly_delta" / f"holo_{alignment_type}", - aln_type=alignment_type, - ) + databases.publish_search_database_bundle( + source_root=overlay_source, + target_root=staging / "weekly_delta" / f"holo_{alignment_type}", + aln_type=alignment_type, ) shadowed = data_dir / "manifests/weekly_shadowed_entries.parquet" if shadowed.is_file(): copyfile(shadowed, staging / "shadowed_entries.parquet") - write_json_atomic( - staging / "manifest.json", - { - "status": "complete", - "bundles": reports, - "overlays": overlay_reports, - "shadowed_entries": ( - "shadowed_entries.parquet" if shadowed.is_file() else None - ), - }, - ) if output.exists(): output.rename(backup) staging.rename(output) @@ -7571,7 +7564,7 @@ def main() -> None: elif args.command == "make-sub-dbs": tasks.make_sub_dbs( data_dir=data_dir, - sub_databases=args.search_dbs or ["holo"], + sub_databases=args.search_dbs or ["holo", "monomer"], cpu=args.threads, scratch_dir=args.scratch_dir.resolve(), ) diff --git a/src/plinder/data/pipeline/tasks.py b/src/plinder/data/pipeline/tasks.py index 7badb1959..1f285c38a 100644 --- a/src/plinder/data/pipeline/tasks.py +++ b/src/plinder/data/pipeline/tasks.py @@ -655,14 +655,20 @@ def make_sub_dbs( """ entries = None identifiers_by_database: dict[str, set[str]] = {} - indexed_search_databases = ( - sub_databases if set(sub_databases) <= {"holo", "apo"} else [] - ) + indexed_search_databases = [] + if set(sub_databases) <= {"holo", "apo", "monomer"}: + indexed_search_databases = list(sub_databases) + elif "monomer" in sub_databases: + indexed_search_databases = ["monomer"] for search_db in indexed_search_databases: chains = ( _protein_scoring_chains(data_dir) if search_db == "holo" - else _apo_scoring_chains(data_dir) + else ( + _monomer_scoring_chains(data_dir) + if search_db == "monomer" + else _apo_scoring_chains(data_dir) + ) ) if search_db == "holo": chain_auth_ids = pd.read_parquet( @@ -684,7 +690,7 @@ def make_sub_dbs( f"{row.entry_pdb_id}_{row.chain_auth_id}" for row in chains.itertuples(index=False) } - if set(sub_databases).difference({"holo", "apo"}): + if set(sub_databases).difference({"holo", "apo", "monomer"}): from plinder.core.scores.entries import entry_views_from_df entries = entry_views_from_df( @@ -717,7 +723,7 @@ def make_sub_dbs( tmp_dir=scratch_dir, threads=cpu, ) - if set(sub_databases).intersection({"holo", "apo"}): + if set(sub_databases).intersection({"holo", "apo", "monomer"}): make_alignment_chain_lookup( data_dir=data_dir, scratch_dir=scratch_dir, @@ -1786,6 +1792,7 @@ def make_ccd_ligand_dbs( threads: int, force_update: bool = False, minimum_similarity: float = 30.0, + build_parity_scores: bool = False, ) -> Path: """Build the CCD-anchored MMP/ECFP4 databases and the ligand-to-CCD match sidecar.""" ccd_ligand_dbs.make_ccd_ligand_dbs( @@ -1794,7 +1801,7 @@ def make_ccd_ligand_dbs( threads=threads, force_update=force_update, minimum_similarity=minimum_similarity, - build_parity_scores=False, + build_parity_scores=build_parity_scores, ) return ccd_ligand_dbs.make_ligand_ccd_match(data_dir=data_dir) @@ -1853,6 +1860,36 @@ def _protein_scoring_chains(data_dir: Path) -> pd.DataFrame: ].copy() +def _monomer_scoring_chains(data_dir: Path) -> pd.DataFrame: + """Return protein chains absent from pocket/interface search targets.""" + chains = pd.read_parquet( + data_dir / "index" / "entry_chains.parquet", + columns=[ + "entry_pdb_id", + "chain_asym_id", + "chain_auth_id", + "chain_receptor_type", + "chain_is_ligand_like", + ], + ) + chains = chains.loc[ + chains["chain_receptor_type"].eq("protein") + & ~chains["chain_is_ligand_like"].fillna(False).astype(bool) + & chains["chain_auth_id"].notna() + ] + holo = _protein_scoring_chains(data_dir)[["entry_pdb_id", "chain_asym_id"]] + return ( + chains.merge( + holo.drop_duplicates(), + on=["entry_pdb_id", "chain_asym_id"], + how="left", + indicator=True, + ) + .loc[lambda rows: rows["_merge"].eq("left_only")] + .drop(columns="_merge") + ) + + def _ligand_apo_query_chains(data_dir: Path) -> pd.DataFrame: """Return proper-ligand protein receptor chains used as apo queries.""" from plinder.data.linked_apo import ligand_holo_chain_keys @@ -2513,6 +2550,22 @@ def alignment_mapping_shard_is_current( return True +def _restore_parquet_schema(path: Path, schema: pa.Schema) -> None: + """Cast a DuckDB-written file back to the shard's original physical types. + + DuckDB always writes text as ``string``, while pandas 3 writes ``large_string``; + the values are identical, so only the physical type is restored. + """ + written = pq.read_schema(path) + if written.equals(schema) or written.names != schema.names: + return + restored = path.with_suffix(path.suffix + ".restored") + with pq.ParquetWriter(restored, schema, compression="zstd") as writer: + for batch in pq.ParquetFile(path).iter_batches(): + writer.write_batch(batch.cast(schema)) + restored.replace(path) + + def _replace_release_alignment_queries( *, old: Path, @@ -2582,6 +2635,7 @@ def _replace_release_alignment_queries( ) finally: connection.close() + _restore_parquet_schema(temporary, old_schema) if not pq.read_schema(temporary).equals(old_schema): raise ValueError( f"patched alignment shard has an unexpected schema: {old}; " diff --git a/src/plinder/data/pipeline/update_release.py b/src/plinder/data/pipeline/update_release.py index 97f005b42..f35d57298 100644 --- a/src/plinder/data/pipeline/update_release.py +++ b/src/plinder/data/pipeline/update_release.py @@ -12,6 +12,7 @@ import os from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from dataclasses import replace +from itertools import groupby from multiprocessing import get_context from pathlib import Path from shutil import copyfile, rmtree @@ -33,7 +34,7 @@ write_json_atomic, ) from plinder.data import clusters, databases -from plinder.data.annotations import get_similarity_scores +from plinder.data.annotations import ccd_ligand_dbs, get_similarity_scores from plinder.data.pipeline import config, score, tasks, utils from plinder.data.pipeline.update_archives import update_ligand_archives from plinder.data.pipeline.update_entries import apply_entry_update @@ -297,25 +298,19 @@ def _queries_with_old_target_hits( ] if not paths: return set() - connection = duckdb.connect() - try: - connection.register( - "affected_targets", - pd.DataFrame({"target_entry": sorted(affected)}), - ) - sources = ", ".join( - f"'{path.as_posix().replace(chr(39), chr(39) * 2)}'" for path in paths - ) - hits = connection.sql( - f""" - SELECT DISTINCT query_entry - FROM read_parquet([{sources}], union_by_name=true) AS alignments - INNER JOIN affected_targets USING (target_entry) - """ - ).df() - finally: - connection.close() - return set(hits["query_entry"].dropna().astype(str)) + target_ids = pa.array(sorted(affected), type=pa.string()) + hits: set[str] = set() + for path in paths: + for batch in pq.ParquetFile(path).iter_batches( + columns=["query_entry", "target_entry"], batch_size=131_072 + ): + mask = pc.is_in(batch.column("target_entry"), value_set=target_ids) + if pc.any(mask).as_py(): + matches = pc.unique(pc.filter(batch.column("query_entry"), mask)) + hits.update( + str(value) for value in matches.to_pylist() if value is not None + ) + return hits def _changed_target_database( @@ -323,46 +318,17 @@ def _changed_target_database( *, search_db: str, affected: set[str], - scratch_dir: Path, - threads: int, ) -> Path | None: chains = _scoring_chains(data_dir, search_db) chains = chains.loc[chains["entry_pdb_id"].astype(str).isin(affected)] - identifiers = { - f"{search_db}_{alignment_type}": _database_identifiers(chains, alignment_type) - for alignment_type in ("foldseek", "mmseqs") - } - if not any(identifiers.values()): - return None - target_dir = scratch_dir / f"changed-{search_db}-targets" - if target_dir.exists(): - rmtree(target_dir) - selected = { - key: ids - for key, ids in identifiers.items() - if ids - and ( - data_dir - / "dbs/weekly_delta" - / key.rsplit("_", 1)[1] - / f"{key.rsplit('_', 1)[1]}.dbtype" - ).is_file() - } - if not selected: + if not any(_database_identifiers(chains, kind) for kind in ("foldseek", "mmseqs")): return None - databases.make_sub_dbs( - target_dir, - { - key: data_dir - / "dbs/weekly_delta" - / key.rsplit("_", 1)[1] - / key.rsplit("_", 1)[1] - for key in selected - }, - identifiers_by_database=selected, - tmp_dir=scratch_dir / f"changed-{search_db}-target-work", - threads=threads, - ) + target_dir = data_dir / "dbs/weekly_delta/subdbs" + if not any( + (target_dir / f"{search_db}_{kind}/exact_cluster.json").is_file() + for kind in ("foldseek", "mmseqs") + ): + raise FileNotFoundError(f"missing changed-target database for {search_db}") return target_dir @@ -582,8 +548,6 @@ def _plan_alignment_repairs( data_dir, search_db=search_db, affected=affected, - scratch_dir=scratch_dir, - threads=threads, ) reverse_hits = _search_changed_targets( data_dir, @@ -673,13 +637,15 @@ def repair_alignments( raw_dir = data_dir / "dbs/subdbs" / f"{search_db}_{alignment_type}" / "aln" for pdb_id in full_queries | targeted_queries: (raw_dir / f"{pdb_id}.parquet").unlink(missing_ok=True) + shadowed_path = data_dir / "manifests/weekly_shadowed_entries.parquet" + shadowed = set( + map(str, pq.read_table(shadowed_path, columns=["pdb_id"])["pdb_id"].to_pylist()) + ) if targeted_queries: changed_targets = _changed_target_database( data_dir, search_db=search_db, affected=affected, - scratch_dir=scratch_dir / "changed-targets", - threads=threads, ) if changed_targets is None: # Obsolete targets have no database to search. Re-search their old @@ -697,15 +663,6 @@ def repair_alignments( changed_targets / f"{search_db}_{backend}/exact_cluster.json" ).is_file() ] - shadowed_path = data_dir / "manifests/weekly_shadowed_entries.parquet" - shadowed = set( - map( - str, - pq.read_table(shadowed_path, columns=["pdb_id"])[ - "pdb_id" - ].to_pylist(), - ) - ) for source_label, query_ids in ( ("base", targeted_queries - shadowed), ("overlay", targeted_queries & shadowed), @@ -739,7 +696,7 @@ def repair_alignments( search_db=search_db, shards=targeted_shards, replacement_query_ids=targeted_queries, - replacement_target_ids=affected, + replacement_target_ids=shadowed, scorer_cfg=selected_cfg, scratch_dir=scratch_dir / f"map-targeted-{search_db}", threads=threads, @@ -750,10 +707,6 @@ def repair_alignments( ) for pdb_id in targeted_queries: (raw_dir / f"{pdb_id}.parquet").unlink(missing_ok=True) - shadowed_path = data_dir / "manifests/weekly_shadowed_entries.parquet" - shadowed = set( - map(str, pq.read_table(shadowed_path, columns=["pdb_id"])["pdb_id"].to_pylist()) - ) for source_label, query_ids in ( ("base", full_queries - shadowed), ("overlay", full_queries & shadowed), @@ -1141,17 +1094,35 @@ def _restore_score_query_caches( if not missing: continue packed = base_data_dir / "scores" / packed_name / f"shard={shard}.parquet" - rows = pq.read_table( - packed, - columns=schema.names, - filters=[("query_entry", "in", list(missing))], - ).cast(schema) - for pdb_id, destination in missing.items(): + rows = ( + pq.read_table( + packed, + columns=schema.names, + filters=[("query_entry", "in", list(missing))], + ) + .cast(schema) + .sort_by([("query_entry", "ascending")]) + ) + seen = set() + offset = 0 + for pdb_id, group in groupby(rows["query_entry"].to_pylist()): + count = sum(1 for _ in group) + selected = rows.slice(offset, count) + offset += count + if pdb_id not in missing: + continue + seen.add(pdb_id) + destination = missing[pdb_id] destination.parent.mkdir(parents=True, exist_ok=True) - selected = rows.filter(pc.equal(rows["query_entry"], pdb_id)) temporary = destination.with_suffix(".parquet.tmp") pq.write_table(selected, temporary, compression="zstd") temporary.replace(destination) + for pdb_id in missing.keys() - seen: + destination = missing[pdb_id] + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(".parquet.tmp") + pq.write_table(rows.slice(0, 0), temporary, compression="zstd") + temporary.replace(destination) def _repair_score_batches( @@ -2134,11 +2105,19 @@ def refresh_ligand_chemistry( ) ccd = None if "make_ccd_ligand_dbs" not in cfg.flow.skip_specific_stages: + ccd_manifest = ( + read_json_cache( + ccd_ligand_dbs.ccd_dbs_dir(data_dir) / "ccd_dbs.manifest.json" + ) + or {} + ) ccd = tasks.make_ccd_ligand_dbs( data_dir=data_dir, scratch_dir=scratch_dir / "ccd", threads=threads, minimum_similarity=float(cfg.ligand.minimum_similarity), + build_parity_scores=bool(ccd_manifest.get("build_parity_scores")) + or ccd_ligand_dbs.ccd_parity_path(data_dir).is_file(), ) return { "status": "complete", @@ -2465,6 +2444,11 @@ def apply_release_update( entries.action.isin(["added", "revised", "obsolete"]), "pdb_id" ].astype(str) ) + if affected and (base / "exports/monomer_similarity_scores").exists(): + raise NotImplementedError( + "weekly updates of releases with monomer similarity pairs require " + "a monomer delta-refresh stage" + ) if report is None: entry_report = apply_entry_update( plan_dir, diff --git a/src/plinder/data/pipeline/updates.py b/src/plinder/data/pipeline/updates.py index b2bff78aa..9982b0a1f 100644 --- a/src/plinder/data/pipeline/updates.py +++ b/src/plinder/data/pipeline/updates.py @@ -218,14 +218,15 @@ def plan_update( threads: int = 8, previous_snapshot: Path | None = None, pdb_ids: Collection[str] | None = None, + refresh_pdb_ids: Collection[str] | None = None, ) -> dict[str, Any]: """Write an update report and work lists into a new, separate directory. Nothing is applied to the release. Blocked plans are review reports, not executable work lists. A partial ``pdb_ids`` scope is useful for testing; omit it for a complete release comparison. Coordinate revisions determine - changes here; validation-only or NextGen enrichment changes without a PDB - revision need a separate annotation refresh. + changes unless ``refresh_pdb_ids`` explicitly requests re-annotation of + unchanged entries against the same source revision. """ data_dir = data_dir.resolve(strict=True) nextgen_root = nextgen_root.resolve(strict=True) @@ -259,6 +260,23 @@ def plan_update( pdb_ids=scope, ) entries = compare_entries(sources, snapshot, read_obsolete(obsolete_path)) + refresh = {normalize_pdb_id(value) for value in refresh_pdb_ids or ()} + missing_refresh = refresh.difference(entries.pdb_id) + if missing_refresh: + raise ValueError( + f"refresh PDB IDs are outside the update scope: {sorted(missing_refresh)}" + ) + unavailable_refresh = refresh.intersection( + entries.loc[entries.action != "unchanged", "pdb_id"] + ) + if unavailable_refresh: + raise ValueError( + f"refresh PDB IDs must be unchanged entries: {sorted(unavailable_refresh)}" + ) + entries.loc[entries.pdb_id.isin(refresh), ["action", "reason"]] = [ + "revised", + "annotation refresh", + ] counts = dict(sorted(Counter(entries.action).items())) changed = bool(entries.action.isin(["added", "revised", "obsolete"]).any()) status = ( @@ -269,6 +287,7 @@ def plan_update( "data_dir": str(data_dir), "nextgen_root": str(nextgen_root), "scope": scope, + "refresh_pdb_ids": sorted(refresh), "inputs": signatures, "counts": counts, "work": { @@ -295,8 +314,8 @@ def plan_update( "Search directions are candidate work, not a complete search-cache repair plan: " "changed representatives and capped hit lists can require full-query searches.", "Recompute covers globally; their representatives and labels can change.", - "This plan compares PDB coordinate revisions, not independent changes to " - "validation reports, NextGen enrichment, CCD, or annotation settings.", + "This plan compares PDB coordinate revisions and explicitly refreshed " + "entries; other annotation changes are not detected automatically.", ], } # Check small catalogues again in case a source changed during revision reads. @@ -339,6 +358,7 @@ def main() -> None: parser.add_argument("--obsolete", type=Path, required=True) parser.add_argument("--previous-snapshot", type=Path) parser.add_argument("--pdb-manifest", type=Path) + parser.add_argument("--refresh-pdb-manifest", type=Path) parser.add_argument("--threads", type=int, default=8) args = parser.parse_args() plan = plan_update( @@ -349,6 +369,11 @@ def main() -> None: threads=args.threads, previous_snapshot=args.previous_snapshot, pdb_ids=load_manifest(args.pdb_manifest) if args.pdb_manifest else None, + refresh_pdb_ids=( + load_manifest(args.refresh_pdb_manifest) + if args.refresh_pdb_manifest + else None + ), ) print(json.dumps(plan, indent=2, sort_keys=True)) if plan["status"] == "blocked": diff --git a/src/plinder/data/pipeline/utils.py b/src/plinder/data/pipeline/utils.py index c07fc1076..eb8e5c75c 100644 --- a/src/plinder/data/pipeline/utils.py +++ b/src/plinder/data/pipeline/utils.py @@ -39,6 +39,9 @@ def get_db_sources( if "holo" in sub_databases or not len(sub_databases): dbs["holo_foldseek"] = data_dir / "dbs" / "foldseek" / "foldseek" dbs["holo_mmseqs"] = data_dir / "dbs" / "mmseqs" / "mmseqs" + if "monomer" in sub_databases: + dbs["monomer_foldseek"] = data_dir / "dbs" / "foldseek" / "foldseek" + dbs["monomer_mmseqs"] = data_dir / "dbs" / "mmseqs" / "mmseqs" if "apo" in sub_databases or not len(sub_databases): dbs["apo_foldseek"] = data_dir / "dbs" / "foldseek" / "foldseek" dbs["apo_mmseqs"] = data_dir / "dbs" / "mmseqs" / "mmseqs" diff --git a/tests/core/test_core_config.py b/tests/core/test_core_config.py index 5ebeb3866..95b046de0 100644 --- a/tests/core/test_core_config.py +++ b/tests/core/test_core_config.py @@ -14,7 +14,8 @@ def test_get_config(): assert cfg == {"data": asdict(ocfg), "context": asdict(ccfg)} -def test_release_identifier_sets_local_and_remote_roots(): +def test_release_identifier_sets_local_and_remote_roots(monkeypatch): + monkeypatch.delenv("PLINDER_MIRROR_URL", raising=False) cfg = config.DataConfig( plinder_release="2026-07", plinder_release_number="1", @@ -23,7 +24,23 @@ def test_release_identifier_sets_local_and_remote_roots(): ) assert cfg.plinder_dir == "/cache/plinder/2026-07/1" - assert cfg.plinder_remote == "https://plinderdata.org/2026-07/1" + assert cfg.plinder_remote == "https://cameo3d.org/plinder/PLINDER-2026-07/1" + + +def test_default_release_uses_cameo_without_environment_overrides(monkeypatch): + for key in ["PLINDER_RELEASE", "PLINDER_RELEASE_NUMBER", "PLINDER_MIRROR_URL"]: + monkeypatch.delenv(key, raising=False) + cfg = config.DataConfig(plinder_mount="/cache") + + assert cfg.plinder_dir == "/cache/plinder/2026-09" + assert cfg.plinder_remote == "https://cameo3d.org/plinder/PLINDER-2026-09" + + +def test_mirror_override_uses_the_file_server_layout(monkeypatch): + monkeypatch.setenv("PLINDER_MIRROR_URL", "https://mirror.example/datasets/") + cfg = config.DataConfig(plinder_release="2026-09", plinder_release_number="") + + assert cfg.plinder_remote == "https://mirror.example/datasets/PLINDER-2026-09" def test_get_config_passed(): diff --git a/tests/core/test_core_scores.py b/tests/core/test_core_scores.py index 3d992eeff..f960dc06b 100644 --- a/tests/core/test_core_scores.py +++ b/tests/core/test_core_scores.py @@ -88,6 +88,22 @@ def similarity_release(tmp_path): "lddt": pd.Series([80], dtype="uint8"), } ).to_parquet(protein_scores / "shard=ab.parquet", index=False) + monomer_scores = exports / "monomer_similarity_scores/alignment_type=mmseqs" + monomer_scores.mkdir(parents=True) + pd.DataFrame( + { + "query_entry": ["3ghi"], + "target_entry": ["1abc"], + "query_chain_mapped": ["C"], + "target_chain_mapped": ["A"], + "source": ["mmseqs"], + "qcov": pd.Series([95], dtype="uint8"), + "tcov": pd.Series([75], dtype="uint8"), + "fident": pd.Series([65], dtype="uint8"), + "seqsim": pd.Series([72], dtype="uint8"), + "lddt": pd.Series([pd.NA], dtype="UInt8"), + } + ).to_parquet(monomer_scores / "part-0.parquet", index=False) alignment_shard = ( tmp_path / "alignments/search_db=holo/alignment_type=mmseqs/shard=ab.parquet" ) @@ -185,6 +201,19 @@ def test_query_protein_similarity_reads_integer_scores(similarity_release): ] +def test_query_protein_similarity_can_include_monomer_pairs(similarity_release): + result = scores.query_protein_similarity( + columns=["query_entry", "target_entry", "fident"], + filters=[("target_entry", "==", "1abc")], + release=similarity_release, + include_monomers=True, + ) + + assert result.to_dict("records") == [ + {"query_entry": "3ghi", "target_entry": "1abc", "fident": 65} + ] + + @pytest.mark.parametrize( ("kind", "expected"), [ diff --git a/tests/core/test_custom_scores.py b/tests/core/test_custom_scores.py index 834e83dd3..fd51d5317 100644 --- a/tests/core/test_custom_scores.py +++ b/tests/core/test_custom_scores.py @@ -23,6 +23,101 @@ ) +def test_custom_chain_similarity_scores_keep_each_alignment(tmp_path): + hits = {} + for backend, rows in { + "mmseqs": [ + ("1abc", "A", 0.8, 0.5, 0.75, "ACDE", "ACDF"), + ("1abc", "A", 0.6, 0.4, 0.5, "AC-E", "ACDE"), + ], + "foldseek": [("2def", "B", 0.9, 0.8, 0.5, "AAAA", "AATA")], + }.items(): + path = tmp_path / f"{backend}.parquet" + frame = pd.DataFrame( + [ + { + "sequence_id": "generated-placeholder", + "query_id": "model2" if backend == "foldseek" else "model", + "query_chain_id": ( + "model2__A" if backend == "foldseek" else "model__A" + ), + "structure_id": "model2" if backend == "foldseek" else "model", + "query_chain_asym_id": "A", + "target_entry": entry, + "target_chain_asym_id": chain, + "source": backend, + "qcov": qcov, + "tcov": tcov, + "fident": fident, + "qaln": qaln, + "taln": taln, + **({"lddt": 0.7} if backend == "foldseek" else {}), + } + for entry, chain, qcov, tcov, fident, qaln, taln in rows + ] + ) + frame.to_parquet(path, index=False) + hits[backend] = path + + manifest = tmp_path / "query_chains.parquet" + pd.DataFrame( + { + "query_id": ["model", "model2"], + "sequence_id": ["original-first", "original-second"], + } + ).to_parquet(manifest, index=False) + output = custom.write_custom_chain_similarity_scores( + hits, + output_path=tmp_path / "chain_scores.parquet", + chain_manifest=manifest, + ) + scores = pd.read_parquet(output) + assert len(scores) == 3 + assert scores["qcov"].tolist() == [80, 60, 90] + assert scores["fident"].tolist() == [75, 50, 50] + assert scores["sequence_id"].tolist() == [ + "original-first", + "original-first", + "original-second", + ] + assert scores["lddt"].isna().tolist() == [True, True, False] + assert scores.iloc[2]["lddt"] == 70 + + empty = tmp_path / "empty.parquet" + pd.DataFrame().to_parquet(empty, index=False) + empty_output = custom.write_custom_chain_similarity_scores( + {"mmseqs": empty}, output_path=tmp_path / "empty_scores.parquet" + ) + assert pd.read_parquet(empty_output).empty + + +def test_custom_chain_similarity_scores_round_half_up(tmp_path): + hits = tmp_path / "hits.parquet" + pd.DataFrame( + { + "query_id": ["model"], + "query_chain_id": ["model__A"], + "structure_id": ["model"], + "query_chain_asym_id": ["A"], + "target_entry": ["1abc"], + "target_chain_asym_id": ["B"], + "source": ["mmseqs"], + "qcov": [0.625], + "tcov": [0.125], + "fident": [0.125], + "qaln": ["ACDE"], + "taln": ["ACDF"], + } + ).to_parquet(hits, index=False) + + output = custom.write_custom_chain_similarity_scores( + {"mmseqs": hits}, output_path=tmp_path / "scores.parquet" + ) + + scores = pd.read_parquet(output) + assert scores.loc[0, ["qcov", "tcov", "fident"]].tolist() == [63, 13, 13] + + def _write_database_prefix(prefix: Path, *, indexed: bool = False) -> None: prefix.parent.mkdir(parents=True, exist_ok=True) prefix.with_suffix(".dbtype").write_bytes(b"db") @@ -53,11 +148,11 @@ def _write_search_bundle(root: Path, backend: str) -> None: "alignment_type": backend, "portable": True, "search_target": "representatives", - "conversion_target": "holo_mmseqs", + "conversion_target": root.name, "cluster_alignments": "cluster_alignments", } _write_database_prefix(root / "representatives", indexed=True) - _write_database_prefix(root / "holo_mmseqs") + _write_database_prefix(root / root.name) _write_database_prefix(root / "cluster_alignments") (root / "exact_cluster.json").write_text(json.dumps(manifest)) @@ -93,6 +188,38 @@ def test_resolve_custom_scoring_assets_accepts_ingest_layout(tmp_path): assert assets.ligand_archives == {"ab": archive} +def test_resolve_custom_scoring_assets_includes_monomer_targets(tmp_path): + _write_index(tmp_path) + for backend in custom.SEARCH_BACKENDS: + _write_search_bundle(tmp_path / "search_databases" / f"holo_{backend}", backend) + _write_search_bundle( + tmp_path / "search_databases" / f"monomer_{backend}", backend + ) + + assets = custom.resolve_custom_scoring_assets( + data_dir=tmp_path, include_monomers=True + ) + assert set(assets.monomer_search_databases) == set(custom.SEARCH_BACKENDS) + assert ( + assets.monomer_search_databases["mmseqs"].conversion_target.name + == "monomer_mmseqs" + ) + + +def test_resolve_custom_scoring_assets_skips_missing_monomer_targets(tmp_path): + _write_index(tmp_path) + for backend in custom.SEARCH_BACKENDS: + _write_search_bundle(tmp_path / "search_databases" / f"holo_{backend}", backend) + _write_search_bundle(tmp_path / "search_databases" / "monomer_mmseqs", "mmseqs") + + assets = custom.resolve_custom_scoring_assets( + data_dir=tmp_path, include_monomers=True + ) + + assert set(assets.search_databases) == set(custom.SEARCH_BACKENDS) + assert set(assets.monomer_search_databases) == {"mmseqs"} + + def test_remote_resolution_requests_only_bounded_assets(tmp_path, monkeypatch): _write_index(tmp_path) for backend in custom.SEARCH_BACKENDS: @@ -117,7 +244,9 @@ def get_plinder_path(*, rel: str) -> Path: "index/alignment_chain_lookup.parquet", "search_databases/holo_foldseek", "search_databases/holo_mmseqs", - "search_databases/manifest.json", + "search_databases/weekly_delta/holo_foldseek", + "search_databases/weekly_delta/holo_mmseqs", + "search_databases/shadowed_entries.parquet", "ligand_archives/xy.parquet", } assert assets.ligand_archives == {"xy": archive} @@ -132,15 +261,6 @@ def test_resolve_custom_scoring_assets_includes_weekly_search_overlay(tmp_path): pd.DataFrame({"pdb_id": ["1abc"]}).to_parquet( tmp_path / "search_databases/shadowed_entries.parquet", index=False ) - (tmp_path / "search_databases/manifest.json").write_text( - json.dumps( - { - "overlays": {"mmseqs": {}}, - "shadowed_entries": "shadowed_entries.parquet", - } - ) - ) - assets = custom.resolve_custom_scoring_assets(data_dir=tmp_path) assert assets.shadowed_entries == {"1abc"} @@ -231,34 +351,21 @@ def get_plinder_path(*, rel: str) -> Path: assert not any(path.startswith("ligand_archives/") for path in requested) -def test_search_database_requires_portable_manifest(tmp_path): +def test_search_database_does_not_require_build_manifest(tmp_path): root = tmp_path / "search_databases" / "holo_foldseek" _write_search_bundle(root, "foldseek") - manifest = json.loads((root / "exact_cluster.json").read_text()) - manifest["portable"] = False - (root / "exact_cluster.json").write_text(json.dumps(manifest)) + (root / "exact_cluster.json").unlink() - with pytest.raises(ValueError, match="not portable"): - custom.resolve_search_database("foldseek", data_dir=tmp_path) + bundle = custom.resolve_search_database("foldseek", data_dir=tmp_path) + assert bundle.search_target == root / "clustered" -@pytest.mark.parametrize( - ("field", "value"), - [ - ("identity", 0.9), - ("coverage", 0.9), - ("coverage_mode", 1), - ("compressed_search_target", True), - ], -) -def test_search_database_requires_exact_cluster_contract(tmp_path, field, value): +def test_search_database_requires_indexed_target(tmp_path): root = tmp_path / "search_databases" / "holo_foldseek" _write_search_bundle(root, "foldseek") - manifest = json.loads((root / "exact_cluster.json").read_text()) - manifest[field] = value - (root / "exact_cluster.json").write_text(json.dumps(manifest)) + (root / "clustered.idx.dbtype").unlink() - with pytest.raises(ValueError, match="exact-cluster contract"): + with pytest.raises(FileNotFoundError, match="indexed"): custom.resolve_search_database("foldseek", data_dir=tmp_path) @@ -903,6 +1010,7 @@ def test_prepare_custom_score_alignments_maps_selected_residues( "tstart": 1, "qcov": 1.0, "fident": 1.0, + "cigar": "2M", "qaln": "AC", "taln": "AC", } @@ -992,6 +1100,7 @@ def test_prepare_custom_protein_score_alignments_reverses_direction( assert result.loc[0, "target_selected_residue_numbers"].tolist() == [custom_number] assert result.loc[0, "selected_residue_identity_bits"] == b"\x01" assert result.loc[0, "fident_qcov"] == pytest.approx(0.75) + assert "cigar" not in result def test_prepare_custom_score_alignments_maps_coordinate_fasta_positions(tmp_path): diff --git a/tests/core/test_r2.py b/tests/core/test_http.py similarity index 51% rename from tests/core/test_r2.py rename to tests/core/test_http.py index 234272d22..9320c22a3 100644 --- a/tests/core/test_r2.py +++ b/tests/core/test_http.py @@ -1,7 +1,4 @@ -import base64 -import gzip -import hashlib -import json +import os from functools import partial from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -11,15 +8,14 @@ import pytest from plinder.core.utils import cpl -from plinder.core.utils import r2 as dataset +from plinder.core.utils import http as dataset @pytest.fixture def mirror(tmp_path, monkeypatch): origin = tmp_path / "origin" - release = origin / "2024-06/v2" + release = origin / "PLINDER-2026-09" release.mkdir(parents=True) - records = [] for key, data in [ ("systems/ab.zip", b"archive"), ("scores/a.parquet", b"one"), @@ -28,39 +24,44 @@ def mirror(tmp_path, monkeypatch): path = release / key path.parent.mkdir(exist_ok=True) path.write_bytes(data) - records.append( - { - "key": key, - "size": len(data), - "md5": base64.b64encode(hashlib.md5(data).digest()).decode(), - } - ) - (release / "manifest.jsonl.gz").write_bytes( - gzip.compress(("\n".join(map(json.dumps, records))).encode()) - ) + os.utime(path, (1700000000, 1700000000)) + requests = [] + + class Handler(SimpleHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_HEAD(self): + requests.append(("HEAD", self.path)) + super().do_HEAD() + + def do_GET(self): + requests.append(("GET", self.path)) + super().do_GET() + server = ThreadingHTTPServer( - ("127.0.0.1", 0), partial(SimpleHTTPRequestHandler, directory=str(origin)) + ("127.0.0.1", 0), partial(Handler, directory=str(origin)) ) thread = Thread(target=server.serve_forever, daemon=True) thread.start() cfg = SimpleNamespace( data=SimpleNamespace( plinder_bucket="plinder", - plinder_release="2024-06", - plinder_release_number="v2", + plinder_release="2026-09", + plinder_release_number="", plinder_dir=str(tmp_path / "cache"), - plinder_remote=f"http://127.0.0.1:{server.server_port}/2024-06/v2", + plinder_remote=f"http://127.0.0.1:{server.server_port}/PLINDER-2026-09", force_update=False, ) ) monkeypatch.setattr(cpl, "get_config", lambda: cfg) monkeypatch.delenv("PLINDER_OFFLINE", raising=False) - cpl.manifest.cache_clear() + cfg.requests = requests + monkeypatch.delenv("PLINDER_OFFLINE_MODE", raising=False) yield cfg, release server.shutdown() server.server_close() thread.join() - cpl.manifest.cache_clear() def test_directory_download_and_truncated_cache_repair(mirror): @@ -73,11 +74,28 @@ def test_directory_download_and_truncated_cache_repair(mirror): assert not (Path(cfg.data.plinder_dir) / "systems/ab.zip").exists() +def test_directory_download_prunes_parquet_files_the_release_dropped(mirror): + cfg, _ = mirror + cache = Path(cfg.data.plinder_dir) + stale = cache / "scores/old.parquet" + stale.parent.mkdir(parents=True) + stale.write_bytes(b"stale") + (cache / "scores/notes.txt").write_text("kept") + elsewhere = cache / "other/x.parquet" + elsewhere.parent.mkdir() + elsewhere.write_bytes(b"kept") + + path = cpl.get_plinder_path(rel="scores") + + assert sorted(p.name for p in path.glob("*.parquet")) == ["a.parquet", "b.parquet"] + assert (path / "notes.txt").read_text() == "kept" + assert elsewhere.read_bytes() == b"kept" + + def test_small_batch_failure_propagates(mirror): - cfg, origin = mirror - (origin / "systems/ab.zip").write_bytes(b"corrupt") - with pytest.raises(ValueError, match="checksum"): - cpl.get_plinder_path(rel="systems/ab.zip") + cfg, _ = mirror + with pytest.raises(FileNotFoundError): + cpl.download_paths(paths=[Path(cfg.data.plinder_dir) / "systems/missing.zip"]) assert not (Path(cfg.data.plinder_dir) / "systems/ab.zip").exists() @@ -95,10 +113,13 @@ def test_path_traversal_rejected(mirror, rel): cpl.get_plinder_path(rel=rel) -def test_unsupported_release_rejected(mirror): +@pytest.mark.parametrize( + "release", [("2024-06", "v2"), ("2026-09", "1"), ("2026-9", ""), ("2026-13", "")] +) +def test_unsupported_release_rejected(mirror, release): cfg, _ = mirror - cfg.data.plinder_release_number = "v1" - with pytest.raises(ValueError, match="2024-06/v2"): + cfg.data.plinder_release, cfg.data.plinder_release_number = release + with pytest.raises(ValueError, match="YYYY-MM"): cpl.get_plinder_path(rel="systems") @@ -107,24 +128,50 @@ def test_missing_file_fails(mirror): cpl.get_plinder_path(rel="missing") -def test_cached_same_size_edit_requires_force_update(mirror, monkeypatch): +def test_same_size_local_edit_preserving_mtime_requires_force_update(mirror): cfg, _ = mirror path = cpl.get_plinder_path(rel="scores/a.parquet") path.write_bytes(b"bad") + os.utime(path, (1700000000, 1700000000)) assert cpl.get_plinder_path(rel="scores/a.parquet").read_bytes() == b"bad" cfg.data.force_update = True assert cpl.get_plinder_path(rel="scores/a.parquet").read_bytes() == b"one" -def test_failed_checksum_preserves_existing_cache(mirror): +def test_same_size_server_hotfix_is_fetched_without_force_update(mirror): cfg, origin = mirror path = cpl.get_plinder_path(rel="scores/a.parquet") - (origin / "scores/a.parquet").write_bytes(b"bad") - cfg.data.force_update = True - with pytest.raises(ValueError, match="checksum"): - cpl.get_plinder_path(rel="scores/a.parquet") - assert path.read_bytes() == b"one" - assert list(path.parent.iterdir()) == [path] + source = origin / "scores/a.parquet" + source.write_bytes(b"new") + os.utime(source, (1700000010, 1700000010)) + + assert not cfg.data.force_update + assert cpl.get_plinder_path(rel="scores/a.parquet").read_bytes() == b"new" + assert path.stat().st_mtime == source.stat().st_mtime + + +def test_directory_listing_refreshes_added_and_removed_files(mirror): + _, origin = mirror + path = cpl.get_plinder_path(rel="scores") + (origin / "scores/a.parquet").unlink() + (origin / "scores/c.parquet").write_bytes(b"new") + + assert cpl.get_plinder_path(rel="scores") == path + assert sorted(p.name for p in path.glob("*.parquet")) == ["b.parquet", "c.parquet"] + + +def test_encoded_nested_paths_and_empty_directory(mirror): + _, origin = mirror + shard = origin / "alignments" / "alignment_type=mmseqs" / "shard=a b%.parquet" + shard.parent.mkdir(parents=True) + shard.write_bytes(b"data") + (origin / "empty").mkdir() + + assert ( + cpl.get_plinder_path(rel="alignments") + / shard.relative_to(origin / "alignments") + ).read_bytes() == b"data" + assert cpl.get_plinder_path(rel="empty").is_dir() def test_native_cloudpath_listing_and_download(mirror, tmp_path): @@ -154,19 +201,15 @@ def test_cache_symlink_escape_is_rejected(mirror, tmp_path): assert not (outside / "a.parquet").exists() -def test_force_update_fetches_manifest_once(mirror, monkeypatch): +def test_force_update_fetches_metadata_once_per_file(mirror): cfg, _ = mirror - actual = dataset.manifest.__wrapped__ - calls = [] - - def tracked(remote): - calls.append(remote) - return actual(remote) - - monkeypatch.setattr(dataset, "manifest", tracked) cfg.data.force_update = True cpl.get_plinder_path(rel="scores") - assert len(calls) == 1 + for name in ["a", "b"]: + assert ( + cfg.requests.count(("HEAD", f"/PLINDER-2026-09/scores/{name}.parquet")) == 1 + ) + assert all("manifest" not in url for _, url in cfg.requests) def test_cached_file_is_not_read_or_transferred(mirror, monkeypatch): @@ -190,12 +233,6 @@ def test_interrupted_native_transfer_restarts(tmp_path, monkeypatch): from http.server import BaseHTTPRequestHandler payload = b"abcdef" * 500000 - row = { - "key": "archive.zip", - "size": len(payload), - "md5": base64.b64encode(hashlib.md5(payload).digest()).decode(), - } - listing = gzip.compress(json.dumps(row).encode()) requests = [] class Handler(BaseHTTPRequestHandler): @@ -204,23 +241,20 @@ def log_message(self, *args): def do_HEAD(self): self.send_response(200) + self.send_header("Content-Length", str(len(payload))) self.end_headers() def do_GET(self): - if self.path.endswith("manifest.jsonl.gz"): - data = listing - else: - requests.append(self.headers.get("Range")) - data = payload + requests.append(self.headers.get("Range")) self.send_response(200) - self.send_header("Content-Length", str(len(data))) + self.send_header("Content-Length", str(len(payload))) self.end_headers() - if data is payload and len(requests) == 1: - self.wfile.write(data[:1500000]) + if len(requests) == 1: + self.wfile.write(payload[:1500000]) self.wfile.flush() self.connection.shutdown(socket.SHUT_WR) else: - self.wfile.write(data) + self.wfile.write(payload) server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) thread = Thread(target=server.serve_forever, daemon=True) @@ -237,25 +271,28 @@ def do_GET(self): server.server_close() -@pytest.mark.parametrize("resource", ["manifest", "object"]) @pytest.mark.parametrize( - "failure", ["503", "stall_headers", "stall_body", "exhausted", "404", "malformed"] + "resource,failure", + [ + (resource, failure) + for resource in ["metadata", "listing", "object"] + for failure in [ + "503", + "stall_headers", + "stall_body", + "exhausted", + "404", + "truncated", + ] + if resource != "metadata" or failure not in {"stall_body", "truncated"} + ], ) def test_download_recovery(tmp_path, monkeypatch, resource, failure): from http.server import BaseHTTPRequestHandler from threading import Event - from urllib.error import HTTPError payload = b"dataset" - listing = gzip.compress( - json.dumps( - { - "key": "data", - "size": len(payload), - "md5": base64.b64encode(hashlib.md5(payload).digest()).decode(), - } - ).encode() - ) + listing = b'Index of /data' release_stall = Event() requests = [] original_timeout = dataset._TimeoutHandler.http_request @@ -273,31 +310,40 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, *args): pass - def do_HEAD(self): - self.send_error(503) # Bootstrap must use GET, not hide this as missing. - - def do_GET(self): - is_manifest = self.path.endswith("manifest.jsonl.gz") - selected = is_manifest == (resource == "manifest") + def respond(self, head=False): + is_listing = self.path == "/" + selected = ( + head + if resource == "metadata" + else not head and is_listing == (resource == "listing") + ) if selected: requests.append(self.path) - fail = selected and (len(requests) == 1 or failure in {"404", "exhausted"}) + fail = selected and ( + len(requests) == 1 or failure in {"404", "exhausted", "truncated"} + ) if fail and failure in {"503", "404", "exhausted"}: self.send_error(503 if failure == "exhausted" else int(failure)) return if fail and failure == "stall_headers": release_stall.wait(5) return - data = listing if is_manifest else payload - if fail and failure == "malformed": - data = b"invalid" # Same length as the object: checksum must reject it. + data = listing if is_listing else payload self.send_response(200) self.send_header("Content-Length", str(len(data))) self.end_headers() + if head: + return if fail and failure == "stall_body": release_stall.wait(5) return - self.wfile.write(data) + self.wfile.write(data[:-1] if fail and failure == "truncated" else data) + + def do_HEAD(self): + self.respond(head=True) + + def do_GET(self): + self.respond() server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) thread = Thread(target=server.serve_forever, daemon=True) @@ -307,17 +353,20 @@ def do_GET(self): def download(): client = dataset.ReleaseClient(f"http://127.0.0.1:{server.server_port}") + if resource == "listing": + list(client.path().iterdir()) client.path("data").download_to(destination) try: - if failure in {"404", "malformed", "exhausted"}: - error = ( - (ValueError, gzip.BadGzipFile) if failure == "malformed" else HTTPError - ) + if failure in {"404", "exhausted", "truncated"}: + from cloudpathlib.exceptions import CloudPathNotExistsError + + error = (OSError, dataset.HTTPException, CloudPathNotExistsError) with pytest.raises(error): download() - assert len(requests) == (3 if failure == "exhausted" else 1) + assert len(requests) == (1 if failure == "404" else 3) assert destination.read_bytes() == b"existing" + assert list(tmp_path.iterdir()) == [destination] else: download() assert len(requests) == 2 @@ -327,4 +376,57 @@ def download(): server.shutdown() server.server_close() thread.join() - dataset.manifest.cache_clear() + + +def test_listing_ignores_navigation_and_escaping_links(mirror, monkeypatch): + cfg, _ = mirror + client = cpl._get_client() + original_open = client.opener.open + + def open_index(request): + response = original_open(request) + if request.full_url.endswith("/scores/"): + from io import BytesIO + + response.read = BytesIO( + b"Index of /scores/" + b'parentsort' + b'external' + b'absoluteescape' + b'encoded escape' + b'ab' + ).read + return response + + monkeypatch.setattr(client.opener, "open", open_index) + assert {p.name for p in client.path("scores").iterdir()} == { + "a.parquet", + "b.parquet", + } + assert all( + "escape" not in url and "example.com" not in url for _, url in cfg.requests + ) + + +def test_invalid_listing_preserves_cached_parquet(mirror, monkeypatch): + cfg, _ = mirror + path = cpl.get_plinder_path(rel="scores") + monkeypatch.setattr(dataset._DirectoryIndex, "handle_data", lambda *args: None) + with pytest.raises(ValueError, match="directory index"): + cpl.get_plinder_path(rel="scores") + assert (path / "a.parquet").read_bytes() == b"one" + assert (path / "b.parquet").read_bytes() == b"two" + + +def test_empty_directory_cannot_prune_through_a_cache_symlink(mirror, tmp_path): + cfg, origin = mirror + (origin / "empty").mkdir() + root = Path(cfg.data.plinder_dir) + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "kept.parquet").write_bytes(b"kept") + (root / "empty").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError): + cpl.get_plinder_path(rel="empty") + assert (outside / "kept.parquet").read_bytes() == b"kept" diff --git a/tests/core/test_reconstruct_scores.py b/tests/core/test_reconstruct_scores.py index 46835472d..c2ad3a6c4 100644 --- a/tests/core/test_reconstruct_scores.py +++ b/tests/core/test_reconstruct_scores.py @@ -65,6 +65,30 @@ def test_prefetch_accepts_one_available_alignment_backend(tmp_path): assert paths == {"1abc": {"foldseek": foldseek}} +def test_prefetch_fetches_protein_score_shards_when_requested(tmp_path, monkeypatch): + requested: list[str] = [] + + def get_plinder_path(*, rel: str) -> Path: + requested.append(rel) + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + return path + + monkeypatch.setattr(release.cpl, "get_plinder_path", get_plinder_path) + + reconstruct.prefetch_similarity_alignments( + ["1abc__1__1.A__1.L"], include_protein_scores=True + ) + + assert requested == [ + "alignment_cigars/search_db=holo/alignment_type=foldseek/shard=ab.parquet", + "exports/protein_similarity_scores/alignment_type=foldseek/shard=ab.parquet", + "alignment_cigars/search_db=holo/alignment_type=mmseqs/shard=ab.parquet", + "exports/protein_similarity_scores/alignment_type=mmseqs/shard=ab.parquet", + ] + + def test_canonical_ligand_resolver_materializes_only_requested_member(tmp_path): archive = tmp_path / "ligand_archives" / "ab.parquet" archive.parent.mkdir(parents=True) diff --git a/tests/core/test_table_query.py b/tests/core/test_table_query.py index e7482e6f3..849986a0e 100644 --- a/tests/core/test_table_query.py +++ b/tests/core/test_table_query.py @@ -436,7 +436,9 @@ def test_annotation_binding_affinity_columns_join_from_sidecar( columns=["ligand_id", "ligand_binding_affinity_endpoint"], release=local_release, ) - assert joined["ligand_binding_affinity_endpoint"].tolist() == [None, "Kd"] + endpoints = joined["ligand_binding_affinity_endpoint"] + assert endpoints.isna().tolist() == [True, False] + assert endpoints.iloc[1] == "Kd" def test_query_table_uses_bound_filter_parameters( diff --git a/tests/data/pipeline/test_public_release.py b/tests/data/pipeline/test_public_release.py new file mode 100644 index 000000000..e44080f7d --- /dev/null +++ b/tests/data/pipeline/test_public_release.py @@ -0,0 +1,145 @@ +from pathlib import Path + +import pytest + +from plinder.core.release import RELEASE_PATHS +from plinder.data.pipeline.public_release import ( + _FILE_ARTIFACTS, + _PARQUET_DIRECTORIES, + _SAMPLING_DIRECTORIES, + _SEARCH_DATABASES, + prepare_public_release, +) + + +def _working_release(root: Path) -> None: + for name in _FILE_ARTIFACTS: + path = root / RELEASE_PATHS[name] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"table") + for name in _PARQUET_DIRECTORIES: + path = root / RELEASE_PATHS.get(name, name) / "part.parquet" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"scores") + (path.parent / "local.json").write_text("/scicore/private/path") + for name in _SAMPLING_DIRECTORIES: + directory = root / RELEASE_PATHS.get(name, name) + cover = directory / "directed_set_cover/metric=pocket_qcov/threshold=50.parquet" + cover.parent.mkdir(parents=True, exist_ok=True) + cover.write_bytes(b"cover") + cache = directory / "directed_set_cover/reductions/part.parquet" + cache.parent.mkdir(parents=True, exist_ok=True) + cache.write_bytes(b"cache") + for name in _SEARCH_DATABASES: + folder = root / "search_databases" / name + folder.mkdir(parents=True) + (folder / "db").write_bytes(b"database") + (folder / "alias").symlink_to("db") + (folder / "local.json").write_text("/scicore/private/path") + (root / "search_databases" / "holo_steam").mkdir() + (root / "search_databases" / "holo_steam" / "db").write_bytes(b"tea") + (root / "scores").mkdir() + (root / "scores" / "temporary.parquet").write_bytes(b"work") + + +def test_public_release_contains_only_reader_artifacts(tmp_path: Path) -> None: + working = tmp_path / "working" + public = tmp_path / "public" + _working_release(working) + + count = prepare_public_release(working, public) + + assert count > len(_FILE_ARTIFACTS) + assert (public / RELEASE_PATHS["annotation_table"]).read_bytes() == b"table" + assert (public / "search_databases/holo_foldseek/alias").read_bytes() == b"database" + assert (public / "search_databases/holo_foldseek/alias").is_symlink() + assert (public / "search_databases/holo_foldseek/db").stat().st_ino == ( + working / "search_databases/holo_foldseek/db" + ).stat().st_ino + assert not list(public.rglob("*.json")) + assert not (public / "search_databases/holo_steam").exists() + assert not (public / "scores").exists() + for name in _SAMPLING_DIRECTORIES: + directory = public / RELEASE_PATHS.get(name, name) + assert ( + directory / "directed_set_cover/metric=pocket_qcov/threshold=50.parquet" + ).read_bytes() == b"cover" + assert not (directory / "directed_set_cover/reductions").exists() + + +def test_public_release_allows_an_empty_monomer_universe(tmp_path: Path) -> None: + working = tmp_path / "working" + _working_release(working) + for name in ("monomer_mmseqs", "monomer_foldseek"): + for path in sorted((working / "search_databases" / name).iterdir()): + path.unlink() + (working / "search_databases" / name).rmdir() + monomer_scores = working / RELEASE_PATHS["monomer_similarity_scores"] + for path in monomer_scores.iterdir(): + path.unlink() + monomer_scores.rmdir() + + prepare_public_release(working, tmp_path / "public") + + assert (tmp_path / "public/search_databases/holo_mmseqs/db").is_file() + assert not (tmp_path / "public/search_databases/monomer_mmseqs").exists() + + +def test_public_release_replacement_drops_old_files(tmp_path: Path) -> None: + working = tmp_path / "working" + public = tmp_path / "public" + _working_release(working) + prepare_public_release(working, public) + (public / "stale.parquet").write_bytes(b"old") + + with pytest.raises(FileExistsError): + prepare_public_release(working, public) + prepare_public_release(working, public, replace=True, copy=True) + + assert not (public / "stale.parquet").exists() + assert (public / "search_databases/holo_foldseek/db").stat().st_ino != ( + working / "search_databases/holo_foldseek/db" + ).stat().st_ino + + +def test_missing_public_file_does_not_replace_release(tmp_path: Path) -> None: + working = tmp_path / "working" + public = tmp_path / "public" + _working_release(working) + prepare_public_release(working, public) + (working / RELEASE_PATHS["annotation_table"]).unlink() + + with pytest.raises(FileNotFoundError, match="annotation_table.parquet"): + prepare_public_release(working, public, replace=True) + + assert (public / RELEASE_PATHS["annotation_table"]).is_file() + + +def test_public_release_rejects_external_database_links(tmp_path: Path) -> None: + working = tmp_path / "working" + public = tmp_path / "public" + _working_release(working) + alias = working / "search_databases/holo_foldseek/alias" + alias.unlink() + alias.symlink_to("../../scores/temporary.parquet") + + with pytest.raises(ValueError, match="link leaves the public release"): + prepare_public_release(working, public) + + assert not public.exists() + + +def test_public_release_rejects_absolute_internal_database_links( + tmp_path: Path, +) -> None: + working = tmp_path / "working" + public = tmp_path / "public" + _working_release(working) + alias = working / "search_databases/holo_foldseek/alias" + alias.unlink() + alias.symlink_to(working / "search_databases/holo_foldseek/db") + + with pytest.raises(ValueError, match="absolute link"): + prepare_public_release(working, public, copy=True) + + assert not public.exists() diff --git a/tests/data/pipeline/test_tasks.py b/tests/data/pipeline/test_tasks.py index 48c886453..8e4526ea9 100644 --- a/tests/data/pipeline/test_tasks.py +++ b/tests/data/pipeline/test_tasks.py @@ -25,6 +25,30 @@ from plinder.data.pipeline.config import LigandConfig +def test_ccd_builder_forwards_parity_selection(tmp_path, monkeypatch): + calls = [] + monkeypatch.setattr( + tasks.ccd_ligand_dbs, + "make_ccd_ligand_dbs", + lambda **kwargs: calls.append(kwargs), + ) + match = tmp_path / "ccd_dbs/ligand_ccd_match.parquet" + monkeypatch.setattr( + tasks.ccd_ligand_dbs, "make_ligand_ccd_match", lambda **_kwargs: match + ) + + assert ( + tasks.make_ccd_ligand_dbs( + data_dir=tmp_path, + scratch_dir=tmp_path / "scratch", + threads=1, + build_parity_scores=True, + ) + == match + ) + assert calls[0]["build_parity_scores"] is True + + def test_foldseek_manifest_signature_reads_references_in_parallel( tmp_path: Path, ) -> None: @@ -1765,6 +1789,9 @@ def test_search_bundle_includes_weekly_overlay(tmp_path, monkeypatch) -> None: base = tmp_path / "dbs/subdbs/holo_mmseqs" base.mkdir(parents=True) + monomer = tmp_path / "dbs/subdbs/monomer_mmseqs" + monomer.mkdir(parents=True) + (monomer / "exact_cluster.json").write_text("{}") overlay = tmp_path / "dbs/weekly_delta/subdbs/holo_mmseqs" overlay.mkdir(parents=True) (overlay / "exact_cluster.json").write_text("{}") @@ -1784,9 +1811,10 @@ def publish_bundle(*, source_root, target_root, aln_type): score.publish_search_database_bundles(tmp_path, alignment_types=["mmseqs"]) - assert sources == [base, overlay] - manifest = json.loads((tmp_path / "search_databases/manifest.json").read_text()) - assert set(manifest["overlays"]) == {"mmseqs"} + assert sources == [base, monomer, overlay] + assert not (tmp_path / "search_databases/manifest.json").exists() + assert (tmp_path / "search_databases/monomer_mmseqs").is_dir() + assert (tmp_path / "search_databases/weekly_delta/holo_mmseqs").is_dir() assert pd.read_parquet(tmp_path / "search_databases/shadowed_entries.parquet")[ "pdb_id" ].tolist() == ["1abc"] @@ -1925,7 +1953,7 @@ def test_protein_scoring_plan_and_alignment_finalization(tmp_path, monkeypatch) assert report["target_clustering"]["expand_to_chain_level"] is True assert report["skipped_queries"] == {} assert (tmp_path / "alignments/manifest.json").is_file() - assert (tmp_path / "search_databases/manifest.json").is_file() + assert not (tmp_path / "search_databases/manifest.json").exists() assert (tmp_path / "search_databases/holo_foldseek/clustered.dbtype").is_file() assert ( tmp_path / "search_databases/holo_mmseqs/cluster_alignments.dbtype" @@ -7281,6 +7309,32 @@ def fake_entry_views(annotation, *, entry_chains): assert lookup_calls == [{"data_dir": tmp_path, "scratch_dir": None, "threads": 1}] +def test_monomer_scoring_chains_excludes_holo_and_ligand_like_chains( + tmp_path, monkeypatch +): + index = tmp_path / "index" + index.mkdir() + pd.DataFrame( + { + "entry_pdb_id": ["1abc"] * 4, + "chain_asym_id": ["A", "B", "C", "D"], + "chain_auth_id": ["A", "B", "C", "D"], + "chain_receptor_type": ["protein", "protein", "protein", "dna"], + "chain_is_ligand_like": [False, False, True, False], + } + ).to_parquet(index / "entry_chains.parquet", index=False) + monkeypatch.setattr( + tasks, + "_protein_scoring_chains", + lambda _data_dir: pd.DataFrame( + {"entry_pdb_id": ["1abc"], "chain_asym_id": ["A"]} + ), + ) + + result = tasks._monomer_scoring_chains(tmp_path) + assert result["chain_asym_id"].tolist() == ["B"] + + def test_make_holo_sub_dbs_selects_protein_receptor_and_interface_chains( tmp_path, monkeypatch ): diff --git a/tests/data/pipeline/test_update_release.py b/tests/data/pipeline/test_update_release.py index e3c3afe42..aa312507f 100644 --- a/tests/data/pipeline/test_update_release.py +++ b/tests/data/pipeline/test_update_release.py @@ -659,6 +659,51 @@ def test_weekly_target_repair_restores_only_needed_query_caches(tmp_path): ).exists() +def test_weekly_target_repair_splits_interleaved_packed_rows_once(tmp_path): + base = tmp_path / "base" + workspace = tmp_path / "workspace" + score_root = base / "dbs/subdbs/search_db=holo" + score_root.mkdir(parents=True) + for pdb_id in ("1abc", "2abc", "3abc"): + pd.DataFrame({"query_system": [f"{pdb_id}__1"]}).to_parquet( + score_root / f"{pdb_id}.parquet", index=False + ) + for folder, schema in ( + ("ligand_3d_candidate_shards", schemas.LIGAND_3D_CANDIDATE_SCHEMA), + ("ligand_pair_score_shards", schemas.LIGAND_PAIR_SCORE_SCHEMA), + ): + packed = base / "scores" / folder / "shard=ab.parquet" + packed.parent.mkdir(parents=True) + pq.write_table( + pa.Table.from_pylist( + [ + {"query_entry": "2abc", "target_entry": "4def"}, + {"query_entry": "1abc", "target_entry": "5def"}, + {"query_entry": "2abc", "target_entry": "6def"}, + ], + schema=schema, + ), + packed, + ) + + update_release._restore_score_query_caches( + base_data_dir=base, + data_dir=workspace, + query_ids={"1abc", "2abc", "3abc"}, + ) + + for folder in ("ligand_3d_candidates", "ligand_pair_scores"): + cache = workspace / "scores" / folder / "search_db=holo/shard=ab" + assert pd.read_parquet(cache / "1abc.parquet")["target_entry"].tolist() == [ + "5def" + ] + assert pd.read_parquet(cache / "2abc.parquet")["target_entry"].tolist() == [ + "4def", + "6def", + ] + assert pd.read_parquet(cache / "3abc.parquet").empty + + def test_ligand_score_export_removes_stale_shards(tmp_path, monkeypatch): shard_dir = tmp_path / "exports" / "ligand_similarity_scores" shard_dir.mkdir(parents=True) @@ -823,10 +868,21 @@ def test_old_target_discovery_reads_retained_cigars(tmp_path): "target_entry": ["9xyz", "8xyz", "9xyz"], } ).to_parquet(cigar, index=False) + foldseek = ( + tmp_path + / "alignment_cigars/search_db=holo/alignment_type=foldseek/shard=cd.parquet" + ) + foldseek.parent.mkdir(parents=True) + pd.DataFrame( + { + "query_entry": ["1abc", "4abc", None], + "target_entry": ["9xyz", "9xyz", "9xyz"], + } + ).to_parquet(foldseek, index=False) assert update_release._queries_with_old_target_hits( tmp_path, search_db="holo", affected={"9xyz"} - ) == {"1abc", "3abc"} + ) == {"1abc", "3abc", "4abc"} def test_changed_target_probe_searches_in_reverse_once_per_backend( @@ -1097,6 +1153,137 @@ def unexpected(*_args, **_kwargs): ) == {"1abc"} +def test_changed_target_database_reuses_weekly_subdb(tmp_path, monkeypatch): + monkeypatch.setattr( + update_release, + "_scoring_chains", + lambda *_args: pd.DataFrame( + { + "entry_pdb_id": ["1abc"], + "chain_asym_id": ["A"], + "chain_auth_id": ["A"], + } + ), + ) + subdb = tmp_path / "dbs/weekly_delta/subdbs" + exact = subdb / "holo_mmseqs/exact_cluster.json" + exact.parent.mkdir(parents=True) + exact.write_text("{}") + + assert ( + update_release._changed_target_database( + tmp_path, + search_db="holo", + affected={"1abc"}, + ) + == subdb + ) + + +def test_targeted_weekly_repair_replaces_cumulative_delta_targets( + tmp_path, monkeypatch +): + manifest = tmp_path / update_release.score.MANIFEST_RELATIVE + manifest.parent.mkdir(parents=True) + pd.DataFrame({"pdb_id": ["1abc", "2def"]}).to_parquet(manifest, index=False) + shadowed = tmp_path / "manifests/weekly_shadowed_entries.parquet" + shadowed.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"pdb_id": ["2def", "3ghi"]}).to_parquet(shadowed, index=False) + target_dir = tmp_path / "changed" + marker = target_dir / "holo_mmseqs/exact_cluster.json" + marker.parent.mkdir(parents=True) + marker.write_text("{}") + monkeypatch.setattr( + update_release, "_changed_target_database", lambda *_args, **_kwargs: target_dir + ) + monkeypatch.setattr(update_release.tasks, "run_batch_searches", lambda **_kw: None) + monkeypatch.setattr( + update_release, "_rebase_unchanged_alignment_manifests", lambda *_a, **_kw: None + ) + replacements = [] + monkeypatch.setattr( + update_release, + "_refresh_alignment_shards", + lambda *_args, **kwargs: replacements.append(kwargs), + ) + + update_release.repair_alignments( + tmp_path, + search_db="holo", + affected={"2def"}, + full_queries=set(), + targeted_queries={"1abc"}, + scorer_cfg=OmegaConf.create({}), + foldseek_cfg=OmegaConf.create({}), + mmseqs_cfg=OmegaConf.create({}), + scratch_dir=tmp_path / "scratch", + threads=1, + batch_size=10, + ) + + assert len(replacements) == 1 + assert replacements[0]["replacement_query_ids"] == {"1abc"} + assert replacements[0]["replacement_target_ids"] == {"2def", "3ghi"} + + +def test_weekly_ccd_refresh_preserves_parity_from_manifest(tmp_path, monkeypatch): + fingerprint_path = tmp_path / "fingerprints/ligands_per_smiles.parquet" + fingerprint_path.parent.mkdir(parents=True) + pd.DataFrame( + { + "ligand_smiles_id": [1], + "ligand_rdkit_canonical_smiles": ["CCO"], + "fingerprint": [b"fingerprint"], + } + ).to_parquet(fingerprint_path, index=False) + ccd_manifest = tmp_path / "ccd_dbs/ccd_dbs.manifest.json" + ccd_manifest.parent.mkdir() + ccd_manifest.write_text('{"build_parity_scores": true}') + monkeypatch.setattr( + update_release.get_similarity_scores, + "ligand_score_manifest_payload", + lambda **_kwargs: {}, + ) + monkeypatch.setattr( + update_release.tasks, "compute_ligand_fingerprints", lambda **_kwargs: None + ) + monkeypatch.setattr( + update_release, "_refresh_chemical_score_shards", lambda **_kwargs: {} + ) + monkeypatch.setattr( + update_release.tasks, "annotate_ligand_similarity", lambda **_kwargs: None + ) + monkeypatch.setattr( + update_release.tasks, "make_ligand_mmp_pairs", lambda **_kwargs: tmp_path + ) + ccd_calls = [] + monkeypatch.setattr( + update_release.tasks, + "make_ccd_ligand_dbs", + lambda **kwargs: ccd_calls.append(kwargs) or tmp_path / "ccd_dbs", + ) + cfg = OmegaConf.create( + { + "ligand": { + "number_id_col": "ligand_smiles_id", + "minimum_similarity": 30.0, + "cofactor_similarity_threshold": 90.0, + }, + "flow": { + "cluster_metrics": [], + "make_ligands_batch_size": 10, + "skip_specific_stages": [], + }, + } + ) + + update_release.refresh_ligand_chemistry( + tmp_path, cfg=cfg, scratch_dir=tmp_path / "scratch", threads=1 + ) + + assert ccd_calls[0]["build_parity_scores"] is True + + def test_release_update_runs_only_configured_search_databases(tmp_path, monkeypatch): plan_dir = tmp_path / "plan" plan_dir.mkdir() diff --git a/tests/data/pipeline/test_updates.py b/tests/data/pipeline/test_updates.py index 9783de3ee..66968bfd7 100644 --- a/tests/data/pipeline/test_updates.py +++ b/tests/data/pipeline/test_updates.py @@ -117,6 +117,25 @@ def test_plan_classifies_entries_and_both_search_directions(inputs, tmp_path): assert before == _bytes(inputs["data_dir"]) +def test_plan_refreshes_unchanged_entry_without_new_revision(inputs, tmp_path): + output = tmp_path / "refresh_plan" + plan = updates.plan_update( + **inputs, + output_dir=output, + pdb_ids=["1abc"], + refresh_pdb_ids=["1ABC"], + ) + assert plan["status"] == "ready" + assert plan["refresh_pdb_ids"] == ["1abc"] + assert load_manifest(output / "ingest.txt") == ["1abc"] + assert load_manifest(output / "invalidate.txt") == ["1abc"] + row = pd.read_parquet(output / "entries.parquet").iloc[0] + assert row.action == "revised" + assert row.reason == "annotation refresh" + assert row.previous_major_revision == row.current_major_revision == 1 + assert row.previous_minor_revision == row.current_minor_revision == 0 + + def test_obsolete_preserves_multiple_successors_and_withdrawals(tmp_path): path = tmp_path / "obsolete.dat" path.write_text( diff --git a/tests/data/test_clusters.py b/tests/data/test_clusters.py index 55b3513df..71d8c0276 100644 --- a/tests/data/test_clusters.py +++ b/tests/data/test_clusters.py @@ -8,6 +8,24 @@ import pytest +def test_slurm_zero_memory_does_not_limit_duckdb(monkeypatch): + from plinder.data.clusters import _set_slurm_memory_limit + + calls = [] + + class Connection: + def execute(self, query, values): + calls.append((query, values)) + + monkeypatch.setenv("SLURM_MEM_PER_NODE", "0") + _set_slurm_memory_limit(Connection(), 3) + assert calls == [] + + monkeypatch.setenv("SLURM_MEM_PER_NODE", "1000") + _set_slurm_memory_limit(Connection(), 4) + assert calls == [("SET memory_limit = ?", ["800MB"])] + + def _component_partition(labels: pd.DataFrame) -> set[frozenset[str]]: return { frozenset(group["ligand_id"].astype(str)) @@ -711,6 +729,89 @@ def test_interface_side_clusters_use_independent_side_nodes(tmp_path): assert edges.loc[("i1::side=1", "i2::side=2"), "similarity"] == pytest.approx(60.0) +def test_interface_clusters_prefer_compact_similarity_tables(tmp_path): + from plinder.data.clusters import ( + prepare_component_node_universe, + prepare_symmetric_edge_plan, + write_symmetric_edge_fragment_batch, + write_symmetric_edge_shard, + ) + + index_dir = tmp_path / "index" + index_dir.mkdir() + pd.DataFrame({"system_id": ["i1", "i2"]}).to_parquet( + index_dir / "interface_annotation_table.parquet", index=False + ) + _write_interface_cluster_universe(index_dir, ["i1", "i2"]) + raw_dir = tmp_path / "interface_scores" + raw_dir.mkdir() + pd.DataFrame( + { + "query_system": ["i1"], + "target_system": ["i2"], + "metric": ["interface_qcov"], + "similarity": [5], + } + ).to_parquet(raw_dir / "shard=ab.parquet", index=False) + exports = tmp_path / "exports" + exports.mkdir() + pd.DataFrame( + { + "query_system": ["i1", "i1", "i2"], + "target_system": ["i2", "i2", "i1"], + "similarity": [80, 75, 60], + } + ).to_parquet(exports / "interface_similarity_scores.parquet", index=False) + pd.DataFrame( + { + "query_half_interface_id": ["i1::side=1", "i2::side=2"], + "target_half_interface_id": ["i2::side=2", "i1::side=1"], + "similarity": [90, 70], + } + ).to_parquet(exports / "interface_half_similarity_scores.parquet", index=False) + + prepare_component_node_universe(tmp_path, entity_type="interface") + plan = prepare_symmetric_edge_plan( + data_dir=tmp_path, + metrics=["interface_qcov", "interface_side_qcov"], + source_batch_size=1, + bucket_count=1, + entity_type="interface", + ) + assert {batch["kind"] for batch in plan["batches"]} == { + "interface_export_interface_qcov", + "interface_export_interface_side_qcov", + } + for batch in plan["batches"]: + write_symmetric_edge_fragment_batch( + data_dir=tmp_path, + batch=batch, + scratch_dir=tmp_path / "scratch-fragments" / batch["key"], + threads=1, + entity_type="interface", + ) + for metric, pair, expected in ( + ("interface_qcov", ("i1", "i2"), 60.0), + ("interface_side_qcov", ("i1::side=1", "i2::side=2"), 70.0), + ): + write_symmetric_edge_shard( + data_dir=tmp_path, + metric=metric, + bucket=0, + scratch_dir=tmp_path / "scratch-shards" / metric, + threads=1, + entity_type="interface", + ) + edges = pd.read_parquet( + tmp_path + / "interface_clusters/symmetric_edges" + / f"metric={metric}/bucket=000.parquet" + ).set_index(["query_node", "target_node"]) + assert edges.loc[pair, "similarity"] == pytest.approx(expected) + if metric == "interface_qcov": + assert edges.loc[pair, "forward_similarity"] == pytest.approx(80.0) + + def test_empty_interface_universe_publishes_typed_empty_cover(tmp_path): from plinder.data.clusters import ( make_directed_cover_component_reduction, diff --git a/tests/data/test_databases.py b/tests/data/test_databases.py index c8cb5197a..7d3ce9546 100644 --- a/tests/data/test_databases.py +++ b/tests/data/test_databases.py @@ -305,3 +305,46 @@ def fake_make_exact_search_db(*, full_db, **_kwargs): assert (target / "exact_cluster.json").is_file() assert original_missing.read_text() == '{"old": true}\n' assert not os.path.samefile(original_missing, db_dir / "missing.json") + + +def test_make_sub_dbs_retires_monomer_db_when_selection_becomes_empty( + tmp_path, monkeypatch +): + full_db = tmp_path / "full" / "mmseqs" + full_db.parent.mkdir() + full_db.with_suffix(".lookup").write_text("0 1abc_A 0\n") + db_dir = tmp_path / "subdbs" + db_dir.mkdir() + clustered: list[str] = [] + + def fake_make_sub_db(ids, _full_db, subdb, _aln_type, *, portable): + database = subdb / subdb.name + _touch_database(database) + database.with_suffix(".index").write_text("".join(f"{i}\t0\t1\n" for i in ids)) + return [] + + def fake_make_exact_search_db(*, full_db, **_kwargs): + clustered.append(full_db.name) + (full_db.parent / "exact_cluster.json").write_text("{}") + return {"status": "complete"} + + monkeypatch.setattr(databases, "make_sub_db", fake_make_sub_db) + monkeypatch.setattr(databases, "make_exact_search_db", fake_make_exact_search_db) + monkeypatch.setattr( + databases, "_completed_exact_search_manifest", lambda *_args: None + ) + target = db_dir / "monomer_mmseqs" + for identifiers in ({"1abc_A"}, set()): + databases.make_sub_dbs( + db_dir, + {"monomer_mmseqs": full_db}, + identifiers_by_database={"monomer_mmseqs": identifiers}, + tmp_dir=tmp_path / "scratch", + ) + if identifiers: + assert (target / "exact_cluster.json").is_file() + + assert clustered == ["monomer_mmseqs"] + assert not (target / "exact_cluster.json").exists() + assert not (target / "selection.json").exists() + assert not list(target.glob("monomer_mmseqs*")) diff --git a/tests/data/test_monomer_similarity.py b/tests/data/test_monomer_similarity.py new file mode 100644 index 000000000..575735ebf --- /dev/null +++ b/tests/data/test_monomer_similarity.py @@ -0,0 +1,200 @@ +from pathlib import Path + +import pandas as pd +import pyarrow.parquet as pq +import pytest + +from plinder.core.utils.schemas import PROTEIN_SIMILARITY_EXPORT_SCHEMA +from plinder.data import monomer_similarity +from plinder.data.monomer_similarity import ( + _chain_ids, + _score_alignment_batches, + finalize_monomer_similarity_scores, +) + + +@pytest.mark.parametrize("backend", ["mmseqs", "foldseek"]) +def test_monomer_pair_scores_map_release_chains(tmp_path: Path, backend: str) -> None: + index = tmp_path / "index" + index.mkdir() + pd.DataFrame( + { + "entry_pdb_id": ["1abc", "2def"], + "chain_auth_id": ["R", "S"], + "chain_asym_id": ["A", "C"], + } + ).to_parquet(index / "alignment_chain_lookup.parquet", index=False) + query = "1abc_R" if backend == "mmseqs" else "pdb_00001abc_xyz-enrich_R" + target = "2def_S" if backend == "mmseqs" else "pdb_00002def_xyz-enrich_MODEL_1_S" + raw = tmp_path / "raw" + raw.mkdir() + pd.DataFrame( + { + "query": [query], + "target": [target], + "qcov": [0.8], + "tcov": [0.6], + "fident": [0.75], + "qaln": ["ACDE"], + "taln": ["ACDF"], + **({"lddt": [0.7]} if backend == "foldseek" else {}), + } + ).to_parquet(raw / "batch.parquet", index=False) + output = tmp_path / "scores.parquet" + assert ( + _score_alignment_batches( + raw, output, backend=backend, chain_ids=_chain_ids(tmp_path, backend) + ) + == 1 + ) + assert pq.read_schema(output).equals(PROTEIN_SIMILARITY_EXPORT_SCHEMA) + row = pd.read_parquet(output).iloc[0] + assert (row.query_entry, row.query_chain_mapped) == ("1abc", "A") + assert (row.target_entry, row.target_chain_mapped) == ("2def", "C") + assert (row.qcov, row.tcov, row.fident) == (80, 60, 75) + assert row.lddt == 70 if backend == "foldseek" else pd.isna(row.lddt) + + +def test_repeated_aligned_sequences_are_scored_once( + tmp_path: Path, monkeypatch +) -> None: + raw = tmp_path / "raw" + raw.mkdir() + alignments = pd.DataFrame( + { + "query": ["1abc_R", "1abc_R"], + "target": ["2def_S", "2def_S"], + "qcov": [0.8, 0.8], + "tcov": [0.6, 0.6], + "fident": [0.75, 0.75], + "qaln": ["ACDE", "ACDE"], + "taln": ["ACDF", "ACDF"], + } + ) + for index in range(len(alignments)): + alignments.iloc[[index]].to_parquet(raw / f"batch-{index}.parquet", index=False) + calls = [] + monkeypatch.setattr( + monomer_similarity, + "get_sequence_similarity_helper", + lambda query, target: calls.append((query, target)) or 0.5, + ) + assert ( + _score_alignment_batches( + raw, + tmp_path / "scores.parquet", + backend="mmseqs", + chain_ids={"1abc_R": ("1abc", "A"), "2def_S": ("2def", "C")}, + ) + == 2 + ) + assert calls == [("ACDE", "ACDF")] + assert pq.ParquetFile(tmp_path / "scores.parquet").num_row_groups == 1 + + +def test_monomer_score_writer_accepts_empty_alignments(tmp_path: Path) -> None: + raw = tmp_path / "raw" + raw.mkdir() + pd.DataFrame( + { + column: pd.Series(dtype=dtype) + for column, dtype in { + "query": "string", + "target": "string", + "qcov": "float64", + "tcov": "float64", + "fident": "float64", + "qaln": "string", + "taln": "string", + }.items() + } + ).to_parquet(raw / "empty.parquet", index=False) + output = tmp_path / "scores.parquet" + assert _score_alignment_batches(raw, output, backend="mmseqs", chain_ids={}) == 0 + assert pq.read_schema(output).equals(PROTEIN_SIMILARITY_EXPORT_SCHEMA) + + +def test_foldseek_missing_lddt_remains_null(tmp_path: Path) -> None: + raw = tmp_path / "raw" + raw.mkdir() + pd.DataFrame( + { + "query": ["pdb_00001abc_xyz-enrich_R"], + "target": ["pdb_00002def_xyz-enrich_S"], + "qcov": [0.8], + "tcov": [0.6], + "fident": [0.75], + "qaln": ["ACDE"], + "taln": ["ACDF"], + "lddt": [None], + } + ).to_parquet(raw / "batch.parquet", index=False) + output = tmp_path / "scores.parquet" + assert ( + _score_alignment_batches( + raw, + output, + backend="foldseek", + chain_ids={ + "pdb_00001abc_xyz-enrich_R": ("1abc", "A"), + "pdb_00002def_xyz-enrich_S": ("2def", "C"), + }, + ) + == 1 + ) + assert pd.isna(pd.read_parquet(output).iloc[0].lddt) + + +def test_finalize_monomer_scores_has_only_alignment_partition(tmp_path: Path) -> None: + staging = tmp_path / "exports/.monomer_similarity_scores_staging" + for backend in ("mmseqs", "foldseek"): + for target_kind in ("holo", "monomer"): + source = ( + staging / f"alignment_type={backend}" / f"target_kind={target_kind}" + ) + source.mkdir(parents=True) + pq.write_table( + PROTEIN_SIMILARITY_EXPORT_SCHEMA.empty_table(), + source / "part-0.parquet", + ) + + assert finalize_monomer_similarity_scores(tmp_path, query_batch_count=1) == 0 + installed = tmp_path / "exports/monomer_similarity_scores" + assert len(list(installed.rglob("*.parquet"))) == 4 + assert not list(installed.rglob("target_kind=*")) + assert set(pd.read_parquet(installed).columns) == { + *PROTEIN_SIMILARITY_EXPORT_SCHEMA.names, + "alignment_type", + } + + +def test_finalize_monomer_scores_requires_every_batch(tmp_path: Path) -> None: + staging = tmp_path / "exports/.monomer_similarity_scores_staging" + staging.mkdir(parents=True) + with pytest.raises(FileNotFoundError): + finalize_monomer_similarity_scores(tmp_path, query_batch_count=1) + assert staging.is_dir() + assert not (tmp_path / "exports/monomer_similarity_scores").exists() + + +def test_finalize_monomer_scores_rejects_extra_or_zero_batches(tmp_path: Path) -> None: + staging = tmp_path / "exports/.monomer_similarity_scores_staging" + for backend in ("mmseqs", "foldseek"): + for target_kind in ("holo", "monomer"): + source = ( + staging / f"alignment_type={backend}" / f"target_kind={target_kind}" + ) + source.mkdir(parents=True) + for index in range(2): + pq.write_table( + PROTEIN_SIMILARITY_EXPORT_SCHEMA.empty_table(), + source / f"part-{index}.parquet", + ) + + with pytest.raises(ValueError, match="positive"): + finalize_monomer_similarity_scores(tmp_path, query_batch_count=0) + with pytest.raises(ValueError, match="unexpected monomer score batches"): + finalize_monomer_similarity_scores(tmp_path, query_batch_count=1) + + assert staging.is_dir() + assert not (tmp_path / "exports/monomer_similarity_scores").exists() diff --git a/tests/test_cif_utils.py b/tests/test_cif_utils.py index e4cf8489c..cbbae6862 100644 --- a/tests/test_cif_utils.py +++ b/tests/test_cif_utils.py @@ -1377,3 +1377,69 @@ def test_drop_self_clashing_symmetry_copies_keeps_lowest_and_distant_copies(): assert sorted(set(kept.sym_id[kept.label_asym_id == "L"])) == [0, 2, 3] assert sorted(set(kept.sym_id[kept.label_asym_id == "P"])) == [0, 2] assert drop_self_clashing_symmetry_copies(kept) is kept + + +def test_drop_overlapping_protein_chains_with_distinct_asym_ids(): + import biotite.structure as struc + + from plinder.data.annotations.cif_utils import drop_self_clashing_symmetry_copies + + base = np.column_stack((np.arange(20) * 3.8, np.zeros(20), np.zeros(20))) + partial = base.copy() + partial[8:, 1] = 10.0 + isolated_errors = base.copy() + isolated_errors[3:, 1] = 15.0 + split_contacts = base.copy() + split_contacts[3:6, 1] = 4.5 + split_contacts[6:, 1] = 30.0 + blocks = [ + ("A", base), + ("B", base.copy()), # identical protein, but a different source asym ID + ("C", partial), # eight clashing C-alpha atoms out of twenty + ("D", base + np.array([0.0, 4.5, 0.0])), # plausible close contact + ("E", base + np.array([0.0, 20.0, 0.0])), + ("F", isolated_errors), # three collisions do not remove a whole chain + ("G", split_contacts), # six collisions across two other chains + ] + atoms = struc.AtomArray(20 * len(blocks)) + atoms.coord = np.concatenate([coord for _, coord in blocks]).astype(np.float32) + atoms.set_annotation("label_asym_id", np.repeat([asym for asym, _ in blocks], 20)) + atoms.set_annotation("sym_id", np.zeros(len(atoms), dtype=int)) + atoms.chain_id = atoms.label_asym_id.copy() + atoms.atom_name = np.repeat("CA", len(atoms)) + atoms.element = np.repeat("C", len(atoms)) + atoms.res_name = np.repeat("ALA", len(atoms)) + atoms.res_id = np.tile(np.arange(1, 21), len(blocks)) + atoms.hetero = np.zeros(len(atoms), dtype=bool) + + kept = drop_self_clashing_symmetry_copies(atoms) + + assert set(kept.label_asym_id) == {"A", "D", "E", "F"} + assert drop_self_clashing_symmetry_copies(kept) is kept + + +def test_drop_long_protein_chain_with_many_absolute_ca_clashes(): + import biotite.structure as struc + + from plinder.data.annotations.cif_utils import drop_self_clashing_symmetry_copies + + base = np.column_stack((np.arange(300) * 3.8, np.zeros(300), np.zeros(300))) + twenty_clashes = base.copy() + twenty_clashes[20:, 1] = 10.0 + nineteen_clashes = base.copy() + nineteen_clashes[19:, 1] = 20.0 + blocks = [("A", base), ("B", twenty_clashes), ("C", nineteen_clashes)] + atoms = struc.AtomArray(300 * len(blocks)) + atoms.coord = np.concatenate([coords for _, coords in blocks]).astype(np.float32) + atoms.set_annotation("label_asym_id", np.repeat([name for name, _ in blocks], 300)) + atoms.set_annotation("sym_id", np.zeros(len(atoms), dtype=int)) + atoms.chain_id = atoms.label_asym_id.copy() + atoms.atom_name = np.repeat("CA", len(atoms)) + atoms.element = np.repeat("C", len(atoms)) + atoms.res_name = np.repeat("ALA", len(atoms)) + atoms.res_id = np.tile(np.arange(1, 301), len(blocks)) + atoms.hetero = np.zeros(len(atoms), dtype=bool) + + kept = drop_self_clashing_symmetry_copies(atoms) + + assert set(kept.label_asym_id) == {"A", "C"} diff --git a/tox.ini b/tox.ini index 8384c91ae..ddbff24b0 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,5 @@ commands = mypy src setenv = PLINDER_LOG_LEVEL=10 PLINDER_OFFLINE=true -passenv = - PLINDER_REGISTRY -extras = test -commands = python flows/docker.py test --dirty {posargs} +allowlist_externals = uv +commands = uv run python -m pytest -n auto {posargs} diff --git a/version.py b/version.py new file mode 100644 index 000000000..dd25366fc --- /dev/null +++ b/version.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, Plinder Development Team +# Distributed under the terms of the Apache License 2.0 +"""Compute the next release tag from "bumpversion" markers in merged PR titles.""" + +from __future__ import annotations + +from subprocess import check_output + + +def get_dev_tag() -> str: + """Return the current ``git describe`` (latest tag, distance, dirty state).""" + return check_output( + ["git", "describe", "--tags", "--always", "--dirty", "--abbrev=8"], + text=True, + ).strip() + + +def get_version_bump(base_tag: str | None = None) -> str: + """ + Inspect the git history since the last tag for "bumpversion {major,minor,skip}" + in a merged PR title and return the new version, defaulting to a patch bump. + An empty string means no release. + """ + if base_tag is None: + base_tag = get_dev_tag().split("-")[0] + bump = "patch" + for token in ["bumpversion major", "bumpversion minor", "bumpversion skip"]: + log = check_output( + [ + "git", + "log", + f"{base_tag}..HEAD", + "--oneline", + "--grep", + token, + "--format=%s", + ], + text=True, + ).strip() + # only count merged PRs, whose titles carry the PR number + if log and "(#" in log: + bump = token.split()[1] + break + if bump == "skip": + return "" + import semver + + return f"v{getattr(semver, f'bump_{bump}')(base_tag.lstrip('v'))}" + + +if __name__ == "__main__": + print(get_version_bump())