From 9408cc24e82fa3a1637c136750ed10e498eedafb Mon Sep 17 00:00:00 2001 From: gautam8387 Date: Mon, 20 Jul 2026 17:31:42 +0200 Subject: [PATCH] update docs --- README.md | 2 +- cyteonto/README.md | 76 ++++++++++++++++++++--------------- docs/FILE_MANAGEMENT.md | 12 ++++-- docs/WORKFLOW.md | 87 +++++++++++++++++++++++------------------ modal_app/README.md | 21 +++++----- tests/README.md | 10 +++-- 6 files changed, 122 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 5de2ac8..68175e2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ 2. Generates a structured description for every label (or part) with an LLM. 3. Embeds those descriptions with a configured embedding model. 4. Matches each embedding to the closest CL term. -5. Scores each author/algorithm pair using an ontology-aware similarity metric (default: a Gaussian kernel on the cosine similarity of the CL term embeddings). Compound pairs use Hungarian bipartite matching with an optional coverage penalty when part counts differ. +5. Scores each author/algorithm pair using an ontology-aware similarity metric (default: a Gaussian kernel on the cosine similarity of the CL term embeddings). Compound pairs default to the maximum entry in the part-by-part score matrix (`compound_scoring="max"`); set `compound_scoring="hungarian_mean"` for Hungarian assignment mean with a coverage penalty when part counts differ. 6. Returns a tidy DataFrame with one row per `(algorithm, pair_index)`. Updated ReadMe: [cyteonto/README.md](cyteonto/README.md). Process flow and file layout: [docs/WORKFLOW.md](docs/WORKFLOW.md), [docs/FILE_MANAGEMENT.md](docs/FILE_MANAGEMENT.md). diff --git a/cyteonto/README.md b/cyteonto/README.md index e70b6db..b0f1f75 100644 --- a/cyteonto/README.md +++ b/cyteonto/README.md @@ -8,7 +8,7 @@ Given two parallel lists of cell type labels (one from the study author, one fro 2. Generates a structured description for each label or part using an LLM. 3. Embeds those descriptions with a configured embedding model. 4. Matches each embedding to the closest CL term via cosine similarity. -5. Scores each author/algorithm pair using an ontology-aware similarity metric (default: kernelised cosine on the CL term embeddings). Compound pairs use Hungarian match mean with a coverage penalty when part counts differ. +5. Scores each author/algorithm pair using an ontology-aware similarity metric (default: kernelised cosine on the CL term embeddings). Compound pairs default to the maximum entry in the part-by-part score matrix (`compound_scoring="max"`); set `compound_scoring="hungarian_mean"` for Hungarian assignment mean with a coverage penalty when part counts differ. 6. Returns a tidy `pandas.DataFrame` with per-pair scores. All LLM descriptions and embeddings are persisted on disk and reused across runs. @@ -98,6 +98,7 @@ df = await cyto.compare( }, run_id="sample_run_001", # optional; auto-generated UUID if omitted metric="cosine_kernel", + compound_scoring="max", # default; use "hungarian_mean" for assignment mean ) print(df) print("run_id used:", df["run_id"].iloc[0]) @@ -122,15 +123,15 @@ One row per `(algorithm, pair_index)`: | `run_id` | str | The `run_id` passed to `compare`. | | `algorithm` | str | Algorithm key from the `algorithms` mapping. | | `pair_index` | int | Position inside the label list, starting at 0. | -| `author_label` | str | The author label for this pair. | -| `algorithm_label` | str | The algorithm label for this pair. | -| `author_ontology_id` | str | Best CL match for the author label. For compound pairs, semicolon-separated ids from the Hungarian assignment only. Empty string if unmatched. | +| `author_label` | str | The author label for this pair (lowercased at compare entry). | +| `algorithm_label` | str | The algorithm label for this pair (lowercased at compare entry). | +| `author_ontology_id` | str | Best CL match per author part, semicolon-separated in part order. Empty string if unmatched. | | `author_ontology_name` | str | Primary CSV label (or OWL fallback) for each id in `author_ontology_id`. | -| `author_embedding_similarity` | float | Mean cosine similarity between author part embeddings and their CL matches. | -| `algorithm_ontology_id` | str | Best CL match for the algorithm label. Same compound rules as author. | +| `author_embedding_similarity` | float \| str | Cosine similarity of each author part to its CL match. Single float when one part; semicolon-separated floats when multiple parts. | +| `algorithm_ontology_id` | str | Best CL match per algorithm part, same rules as author. | | `algorithm_ontology_name` | str | Names for ids in `algorithm_ontology_id`. | -| `algorithm_embedding_similarity` | float | Mean cosine similarity between algorithm part embeddings and their CL matches. | -| `cytescore_similarity` | float | Score under the chosen `metric`; `0.0` if either side is unmatched. | +| `algorithm_embedding_similarity` | float \| str | Same shape as author embedding similarity, for algorithm parts. | +| `cytescore_similarity` | float | Score under the chosen `metric` / compound reducer; `0.0` if either side is unmatched. | | `similarity_method` | str | `cytescore`, `cytescore_compound`, `string_similarity`, `partial_match`, `no_matches`, or `empty`. | --- @@ -145,7 +146,7 @@ Read once on import, via `python-dotenv`: |-----------------------------|-----------------------------------------------------------|---------| | `EMBEDDING_MODEL_API_KEY` | Fallback `apiKey` when `EmbdConfig.apiKey` is `None`. | `""` | | `NCBI_API_KEY` | Optional. Raises PubMed rate limits for the agent tool. | `""` | -| `CYTEONTO_LOG_LEVEL` | Loguru log level (`DEBUG`, `INFO`, `WARNING`, ...). | `INFO` | +| `CYTEONTO_LOG_LEVEL` | Loguru log level (`DEBUG`, `INFO`, `WARNING`, ...). | `DEBUG` | | `CYTEONTO_LOG_FILE` | Optional log file path with 10 MB rotation. | unset | The LLM agent keys follow whatever the caller passes to `pydantic_ai`; this package does not manage them. @@ -238,7 +239,7 @@ cyto = await CyteOnto.from_config( Additional behavior on top of `__init__`: -1. Checks that `cell_ontology/cell_to_cell_ontology.csv` and `cell_ontology/cl.owl` exist. +1. Checks that `cell_ontology/cell_to_cell_ontology.csv` (and preferably the enriched CSV) and `cell_ontology/cl.owl` exist. Lookups prefer `cell_to_cell_ontology_enriched.csv` when present. 2. Computes ontology paths from `llm.to_artifact_key()` and `embedding.to_artifact_key()`. 3. If `force_regenerate=True`, unlinks both the ontology embeddings NPZ and the descriptions JSON before proceeding. 4. Loads the descriptions JSON (if present) and drops any blank entries so they can be retried. @@ -259,6 +260,7 @@ df = await cyto.compare( metric_params: dict[str, Any] | None = None, min_match_similarity: float = 0.1, use_cache: bool = True, + compound_scoring: Literal["max", "hungarian_mean"] = "max", ) ``` @@ -268,33 +270,35 @@ Constraints: - Every value in `algorithms` must be the same length as `author_labels`. Mismatched lengths raise `ValueError`. - Algorithm names must be unique. `"author"` is reserved and rejected. +- `compound_scoring` must be `"max"` or `"hungarian_mean"`; any other value raises `ValueError`. +- Non-empty author and algorithm labels (and decomposed parts) are lowercased before decompose/describe/embed so casing does not split caches. - `run_id` is used as a cache namespace on disk. Reusing the same id reuses cached author and algorithm embeddings when the labels match. If you pass `None`, a UUID of the form `run-` is generated and logged; the same value is written into every result row so you can recover it later. Call flow: -1. `_resolve_label_parts` decomposes unique labels via LLM (cached per `run_id` under `decompositions/`). -2. `_embed_user_labels` on the union of sanitized parts per side (author, then each algorithm). -3. `_match` returns the best CL id and similarity for each part. -4. `_ensure_similarity()` lazy-loads the OWL file and the ontology embeddings into `OntologySimilarity`. -5. For each aligned pair index: +1. Lowercase non-empty labels on both sides. +2. `_resolve_label_parts` decomposes unique labels via LLM (cached per `run_id` under `decompositions/`). Compound parts are lowercased. +3. `_embed_user_labels` on the union of sanitized parts per side (author, then each algorithm). +4. `_match` returns the best CL id and similarity for each part. +5. `_ensure_similarity()` lazy-loads the OWL file and the ontology embeddings into `OntologySimilarity`. +6. For each aligned pair index: - **Simple pair** (one part on each side): single `OntologySimilarity.similarity` call; `similarity_method` from `_method_for`. - - **Compound pair** (more than one part on either side): build an m×n score matrix, run Hungarian max-weight matching, average assigned scores; multiply by `min(m,n)/max(m,n)` when `m ≠ n`; `similarity_method = cytescore_compound`. -6. Rows are concatenated into a DataFrame with columns from `RESULT_COLUMNS`. + - **Compound pair** (more than one part on either side): build an m×n score matrix `S`, then reduce with `compound_scoring` (see below); `similarity_method = cytescore_compound`. +7. Rows are concatenated into a DataFrame with columns from `RESULT_COLUMNS`. Ontology ids/names and embedding similarities list **all** parts in order (not only Hungarian-matched pairs). ### Compound label scoring Mixture labels such as doublets or mixed populations are poor matches when embedded as a single string. `compare` therefore: -1. Calls `decompose_labels` to split a label into one or more cell-type parts (semicolon-separated synonyms stay as one part). +1. Calls `decompose_labels` to split a label into one or more cell-type parts (semicolon-separated synonyms stay as one part; compound part names are lowercased). 2. Embeds and matches each unique part. -3. Scores compound pairs with **Hungarian match mean**: - - Build matrix `S` where `S[i,j]` is the cytescore between author part `i` and algorithm part `j`. - - Pick `k = min(m,n)` one-to-one assignments that maximize total score. - - `match_mean` = mean of assigned cell scores. - - If `m ≠ n`, multiply by coverage `min(m,n) / max(m,n)`. -4. Writes `similarity_method = cytescore_compound`. Ontology ids and names list only the matched assignment pairs (semicolon-separated when `k > 1`). +3. Builds matrix `S` where `S[i,j]` is the cytescore between author part `i` and algorithm part `j`. +4. Reduces `S` with `compound_scoring`: + - **`"max"` (default):** `cytescore_similarity = max(S)`. + - **`"hungarian_mean"`:** pick `k = min(m,n)` one-to-one assignments that maximize total score; take the mean of those `k` scores; if `m ≠ n`, multiply by coverage `min(m,n) / max(m,n)`. +5. Writes `similarity_method = cytescore_compound`. Result ontology ids/names and per-part embedding similarities include every part on that side. -Illustrative scores (see `notebooks/quick_tutorial.ipynb`): +Illustrative scores with `compound_scoring="hungarian_mean"` (see `notebooks/quick_tutorial.ipynb`): | Scenario | m×n | Typical score | |----------|-----|---------------| @@ -304,6 +308,8 @@ Illustrative scores (see `notebooks/quick_tutorial.ipynb`): | Partial overlap, extra author type | 3×2 | ~0.35 | | Author doublet vs single type | 2×1 | best match × 0.5 | +With the default `"max"`, the same matrices report the highest single cell of `S` instead (for example 2×1 with scores `[0.6, 0.2]` yields `0.6`). + ### `compare_anndata` (async) Pulls label lists out of `adata.obs[target_columns]` for each AnnData object and delegates to `compare`. Skips any object missing `author_column` and warns for missing target columns. @@ -353,7 +359,7 @@ CyteOnto.compare ├─ _match(algo_part_emb) └─ for each pair_index: ├─ simple: OntologySimilarity.similarity(a_id, g_id, ...) - └─ compound: build S, _hungarian_match_mean(S) → cytescore_compound + └─ compound: build S; max(S) or _hungarian_match_mean(S) → cytescore_compound ``` --- @@ -418,7 +424,7 @@ Usage limits default to `request_limit=50, input_tokens_limit=60_000`. Override ### Compound label decomposition -`describe.decompose_label` and `describe.decompose_labels` call a dedicated LLM agent with `output_type=LabelDecomposition`. The model decides whether a label names multiple cell types (doublets, mixed populations) and returns sanitized parts. Semicolon-separated synonyms stay as one part. Results are cached per `run_id` under `user_files/decompositions//decompositions_.json`. +`describe.decompose_label` and `describe.decompose_labels` call a dedicated LLM agent with `output_type=LabelDecomposition`. The model decides whether a label names multiple cell types (doublets, mixed populations) and returns sanitized parts in lowercase. Semicolon-separated synonyms stay as one part. Results are cached per `run_id` under `user_files/decompositions//decompositions_.json`. Tune description batching by editing the constants at the top of `describe.py` or by passing `second_pass_wait_seconds` explicitly; the other values are module-level for now. @@ -448,7 +454,7 @@ Tune description batching by editing the constants at the top of `describe.py` o |----------------|-------------|-------| | `initialLabel` | `str` | Input label, copied verbatim. | | `isCompound` | `bool` | `true` when the label names more than one cell type. | -| `parts` | `list[str]` | Sanitized cell-type parts. For non-compound labels, a single element equal to the label. | +| `parts` | `list[str]` | Sanitized cell-type parts (lowercased when compound). For non-compound labels, a single element equal to the label. | Used only for decomposition; descriptions are generated per part afterward. @@ -541,8 +547,9 @@ Set via `PathConfig(data_dir=..., user_dir=...)`: ``` / ├── cell_ontology/ -│ ├── cell_to_cell_ontology.csv shipped -│ └── cl.owl shipped +│ ├── cell_to_cell_ontology.csv shipped (original synonyms) +│ ├── cell_to_cell_ontology_enriched.csv shipped or built by setup.py +│ └── cl.owl shipped └── embedding/ ├── cell_ontology/ │ └── embeddings__.npz @@ -563,6 +570,10 @@ Set via `PathConfig(data_dir=..., user_dir=...)`: └── decompositions_.json ``` +`OntologyMapping` prefers the enriched CSV when it exists. That file keeps the original `label` column for display and adds `label_normalized` (lowercase) for lookup and ontology description inputs. Rows that share the same `(ontology_id, label_normalized)` after lowercasing are deduplicated (first kept). + +`uv run python cyteonto/setup.py` downloads the original CSV, OWL, primary ontology descriptions/embeddings (required), backup pair artifacts (optional; failures warn and continue), and the enriched CSV (optional; builds locally from the original if the CDN file is missing). + Filename rules (`ModelArtifactKey.filename_segment`, `paths._clean_identifier`): - Artifact key segment: `{provider}_{company}-{modelName}` after sanitizing `/`, `:`, and spaces. @@ -601,6 +612,7 @@ When `use_cache=False` is passed to `compare`, all cache lookups are skipped and - Ontology embedding generation failure: `from_config` raises `RuntimeError`. - User embedding generation failure: `compare` raises `RuntimeError` with the offending identifier. - Label length mismatch between author and algorithm lists: `compare` raises `ValueError`. +- Invalid `compound_scoring`: `compare` raises `ValueError`. - Duplicate algorithm name or an algorithm named `"author"`: `compare` raises `ValueError`. - Ontology match below `min_match_similarity`: CL id stored as empty string, `similarity_method` becomes `partial_match` or `no_matches`. - Both labels empty on a pair: `similarity_method = empty`, ontology fields blank. @@ -616,7 +628,7 @@ When `use_cache=False` is passed to `compare`, all cache lookups are skipped and - **New provider**: add a URL to `_PROVIDER_URL` in `embed.py`, extend the `EmbdProvider` `Literal` in `models.py`, and adjust `_headers` / `_build_payload` / `_extract_embedding` if the request or response shape differs. - **New similarity metric**: implement a helper in `OntologySimilarity`, add a branch in `OntologySimilarity.similarity`, and update the table above. - **Alternative prompt**: edit `describe._build_prompt`. The existing prompt lists every `CellDescription` field with a soft length cap and a neutral tone; keep the same structure to avoid anchoring bias. -- **Different ontology file**: pass a custom `data_dir`. The package expects the two files under `/cell_ontology/` to be named exactly `cl.owl` and `cell_to_cell_ontology.csv`. +- **Different ontology file**: pass a custom `data_dir`. The package expects `cl.owl` and `cell_to_cell_ontology.csv` under `/cell_ontology/`. Prefer also shipping or generating `cell_to_cell_ontology_enriched.csv` via `setup.py`. --- @@ -626,7 +638,7 @@ Already declared in the project `pyproject.toml`: - `pydantic`, `pydantic-ai` - `aiohttp`, `tenacity` -- `numpy`, `pandas`, `scikit-learn` +- `numpy`, `pandas`, `scikit-learn` (Hungarian assignment uses `scipy.optimize`, pulled in transitively) - `owlready2` - `loguru`, `python-dotenv`, `tqdm` - `requests` (PubMed tool) diff --git a/docs/FILE_MANAGEMENT.md b/docs/FILE_MANAGEMENT.md index 7991b77..9acc8c9 100644 --- a/docs/FILE_MANAGEMENT.md +++ b/docs/FILE_MANAGEMENT.md @@ -9,8 +9,9 @@ Default roots: `cyteonto/data/` for shipped and generated ontology data, `cyteon ``` / ├── cell_ontology/ -│ ├── cl.owl # shipped -│ └── cell_to_cell_ontology.csv # shipped +│ ├── cl.owl # shipped +│ ├── cell_to_cell_ontology.csv # shipped (original synonyms) +│ └── cell_to_cell_ontology_enriched.csv # shipped or built by setup.py └── embedding/ ├── cell_ontology/ │ └── embeddings__.npz @@ -39,12 +40,15 @@ Default roots: `cyteonto/data/` for shipped and generated ontology data, `cyteon `` and algorithm names are passed through `_clean_identifier` (e.g. `sample.run.001` becomes `sample_run_001`). Model names use `_clean_model` (slashes and colons become hyphens; case and dots are preserved). +The enriched CSV keeps original `label` values for display and adds `label_normalized` (lowercase) for lookups. `OntologyMapping` prefers the enriched file when present. `setup.py` downloads it from the CDN when available, or builds it locally from the original CSV (deduplicating identical `(ontology_id, label_normalized)` rows). + ## File naming ### Ontology (generated once per model configuration) | Artifact | Pattern | Example | |----------|---------|---------| +| Enriched mapping | `cell_to_cell_ontology_enriched.csv` | columns: `ontology_id`, `label`, `label_normalized` | | Descriptions | `descriptions_.json` | `descriptions_together_moonshotai-Kimi-K2.6.json` | | Embeddings | `embeddings__.npz` | `embeddings_together_moonshotai-Kimi-K2.6_openrouter_qwen-qwen3-embedding-8b.npz` | @@ -74,7 +78,7 @@ cyto = await CyteOnto.from_config( ) ``` -Custom `data_dir` must still contain `cell_ontology/cl.owl` and `cell_ontology/cell_to_cell_ontology.csv` with those exact filenames. +Custom `data_dir` must still contain `cell_ontology/cl.owl` and `cell_ontology/cell_to_cell_ontology.csv` with those exact filenames. Prefer also providing `cell_to_cell_ontology_enriched.csv` (via `setup.py` or CDN). ## Caching behavior @@ -88,7 +92,7 @@ Custom `data_dir` must still contain `cell_ontology/cl.owl` and `cell_ontology/c Blank LLM failures are not written to JSON; those labels are retried on the next call. Embeddings for those positions may still use the raw label text so array shape stays aligned. -Per-label caching means adding one new label to a run does not re-describe existing labels. +Per-label caching means adding one new label to a run does not re-describe existing labels. Compare lowercases non-empty labels before caching, so `"Plasma cell"` and `"plasma cell"` share the same describe/embed/decompose keys. Set `use_cache=False` on `compare` to bypass all of the above for that invocation. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index e3e8f11..60b6964 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -4,34 +4,42 @@ CyteOnto compares parallel lists of cell type labels (author reference vs one or ## End-to-end flow -1. **Setup (once per model pair)** — `await CyteOnto.from_config(agent, embedding, llm)` ensures CL term descriptions and ontology embeddings exist on disk. -2. **Compare (per analysis)** — `await cyto.compare(author_labels, algorithms={...}, run_id=...)` decomposes mixture labels when needed, describes and embeds label parts, matches them to CL terms, and scores each author/algorithm pair. +1. **Setup (once per environment)** — `uv run python cyteonto/setup.py` downloads CL assets and precomputed ontology descriptions/embeddings. Then `await CyteOnto.from_config(agent, embedding, llm)` ensures the active model pair is ready (generates missing descriptions/embeddings if needed). +2. **Compare (per analysis)** — `await cyto.compare(author_labels, algorithms={...}, run_id=...)` lowercases labels, decomposes mixture labels when needed, describes and embeds label parts, matches them to CL terms, and scores each author/algorithm pair. 3. **Persist and reuse** — Descriptions (JSON), embeddings (NPZ), and label decompositions (JSON) are written under `user_files/` keyed by `run_id`, so reruns with the same labels skip redundant LLM calls. -## Setup: `from_config` +## Setup: `setup.py` and `from_config` -Runs once when you construct a ready-to-use instance: +**`setup.py`** (CDN download): -1. Verify `cell_ontology/cell_to_cell_ontology.csv` and `cell_ontology/cl.owl` exist under `data_dir`. -2. Load or generate LLM descriptions for every CL term (per text model). -3. Embed those descriptions and save an ontology NPZ (per text + embedding model pair). +1. Required: `cell_to_cell_ontology.csv`, `cl.owl`, primary LLM descriptions JSON, primary embedding NPZ. +2. Optional backup pair artifacts: failures log a warning and setup continues with the primary pair only. +3. Enriched CSV: download `cell_to_cell_ontology_enriched.csv` when available; otherwise build it locally from the original CSV (`label_normalized` = lowercase `label`, drop duplicate `(ontology_id, label_normalized)` rows). -If both artifacts already exist and every CL id has a non-blank description, setup returns immediately. Use `force_regenerate=True` to delete and rebuild the ontology cache. +**`from_config`**: + +1. Prefer `cell_to_cell_ontology_enriched.csv` for mapping when present; otherwise load the original CSV and normalize labels in memory. +2. Verify `cl.owl` exists under `data_dir`. +3. Load or generate LLM descriptions for every CL term (per text model). +4. Embed those descriptions and save an ontology NPZ (per text + embedding model pair). + +If both ontology artifacts already exist and every CL id has a non-blank description, `from_config` returns immediately. Use `force_regenerate=True` to delete and rebuild the ontology cache. ## Compare: `compare` For each call: 1. **Resolve `run_id`** — Use the value you pass, or an auto-generated `run-` (logged at INFO and stored in every result row). -2. **Decompose labels** — Unique non-empty labels on each side are passed through `decompose_labels`. Mixture labels (doublets, mixed populations) split into cell-type parts; simple labels map to a single part. Cached under `user_files/decompositions//`. -3. **Describe and embed parts** — Load cached descriptions/embeddings when possible; generate missing ones via LLM; embed unique parts; save under `user_files/...//author/` and per-algorithm paths. -4. **Match to CL** — Cosine similarity between part embeddings and the precomputed ontology embedding matrix. Matches below `min_match_similarity` (default `0.1`) are treated as unmatched (empty ontology id). -5. **Per algorithm** — Repeat decompose/describe/embed/cache/match for each algorithm label list (same length as `author_labels`). -6. **Pair scoring** — For each aligned index: +2. **Lowercase labels** — Non-empty author and algorithm labels are lowercased so casing does not split describe/embed/decompose caches. Result columns echo the lowercased strings. +3. **Decompose labels** — Unique non-empty labels on each side are passed through `decompose_labels`. Mixture labels (doublets, mixed populations) split into lowercase cell-type parts; simple labels map to a single part. Cached under `user_files/decompositions//`. +4. **Describe and embed parts** — Load cached descriptions/embeddings when possible; generate missing ones via LLM; embed unique parts; save under `user_files/...//author/` and per-algorithm paths. +5. **Match to CL** — Cosine similarity between part embeddings and the precomputed ontology embedding matrix. Matches below `min_match_similarity` (default `0.1`) are treated as unmatched (empty ontology id). +6. **Per algorithm** — Repeat decompose/describe/embed/cache/match for each algorithm label list (same length as `author_labels`). +7. **Pair scoring** — For each aligned index: - **Simple pair** (one part on each side): `OntologySimilarity.similarity(...)` when both parts matched; otherwise `0.0` with `partial_match` or `no_matches`. - - **Compound pair** (more than one part on either side): build an m×n cytescore matrix, Hungarian max-weight matching, mean of assigned scores; multiply by `min(m,n)/max(m,n)` when `m ≠ n`; `similarity_method = cytescore_compound`. + - **Compound pair** (more than one part on either side): build an m×n cytescore matrix `S`, then reduce with `compound_scoring` (default `"max"` → `max(S)`; `"hungarian_mean"` → Hungarian assignment mean with coverage when `m ≠ n`); `similarity_method = cytescore_compound`. - **Empty labels**: both empty → `empty`; one empty → non-empty side keeps ontology fields, score `0.0`, `similarity_method = empty`. -7. **Results** — A `pandas.DataFrame` with one row per `(algorithm, pair_index)`. +8. **Results** — A `pandas.DataFrame` with one row per `(algorithm, pair_index)`. Pass `use_cache=False` to skip on-disk lookups and regenerate everything for that call. @@ -42,11 +50,11 @@ Pass `use_cache=False` to skip on-disk lookups and regenerate everything for tha | `run_id` | Namespace used for caches and result tagging | | `algorithm` | Key from the `algorithms` mapping | | `pair_index` | Index into the parallel label lists (0-based) | -| `author_label`, `algorithm_label` | Raw input strings | -| `author_ontology_id`, `algorithm_ontology_id` | Best CL match per part; semicolon-separated for compound pairs (matched assignment only). Empty string if unmatched. | +| `author_label`, `algorithm_label` | Input strings after compare-time lowercasing | +| `author_ontology_id`, `algorithm_ontology_id` | Best CL match per part, semicolon-separated in part order. Empty string if unmatched. | | `author_ontology_name`, `algorithm_ontology_name` | Primary CSV label (or OWL fallback) for each id above | -| `author_embedding_similarity`, `algorithm_embedding_similarity` | Mean cosine similarity of parts to their CL matches | -| `cytescore_similarity` | Score from the chosen metric when applicable; else `0.0` | +| `author_embedding_similarity`, `algorithm_embedding_similarity` | Per-part cosine similarity to the matched CL term. Single float when one part; semicolon-separated floats when multiple parts. | +| `cytescore_similarity` | Score from the chosen metric / compound reducer when applicable; else `0.0` | | `similarity_method` | How the row was classified (see below) | ### `similarity_method` values @@ -54,7 +62,7 @@ Pass `use_cache=False` to skip on-disk lookups and regenerate everything for tha | Value | When | |-------|------| | `cytescore` | Simple pair; both parts matched valid `CL:` ids; hierarchy/embedding metric applied | -| `cytescore_compound` | Compound pair; Hungarian match mean (with coverage penalty when part counts differ) | +| `cytescore_compound` | Compound pair; score reduced with `compound_scoring` (`max` or `hungarian_mean`) | | `partial_match` | Exactly one side matched the ontology | | `no_matches` | Neither side matched | | `string_similarity` | Both ids present but not standard `CL:` prefixes (rare) | @@ -64,13 +72,13 @@ Pass `use_cache=False` to skip on-disk lookups and regenerate everything for tha When either side has more than one part after decomposition: -1. Score every author-part vs algorithm-part pair with the chosen `metric`. -2. Select `k = min(m,n)` assignments that maximize total score (Hungarian algorithm). -3. `match_mean` = mean of the `k` assigned scores. -4. If `m ≠ n`, multiply by coverage `min(m,n) / max(m,n)`. -5. Ontology ids and names in the result list only the matched pairs. +1. Score every author-part vs algorithm-part pair with the chosen `metric` → matrix `S`. +2. Reduce `S` with `compound_scoring`: + - **`max` (default):** `cytescore_similarity = max(S)`. + - **`hungarian_mean`:** select `k = min(m,n)` assignments that maximize total score; mean those `k` scores; if `m ≠ n`, multiply by coverage `min(m,n) / max(m,n)`. +3. Ontology ids, names, and embedding similarities in the result list **all** parts on that side (not only Hungarian-matched pairs). -See `notebooks/quick_tutorial.ipynb` for worked examples (2×2 same compound ~1.0, 3×2 partial overlap ~0.35). +See `notebooks/quick_tutorial.ipynb` for worked examples. ## Run organization @@ -86,14 +94,14 @@ Identifiers are normalized for paths (`/`, `:`, spaces, `.` replaced) — see [F The default `metric="cosine_kernel"` applies a Gaussian hill on raw embedding cosine between the two matched CL term vectors. Other options include `cosine_direct`, OWL hierarchy metrics (`path`, `set:jaccard`, ...), and `simple` (string fallback on CL id strings). See the metrics table in [cyteonto/README.md](../cyteonto/README.md#similarity-metrics). -For simple pairs, `cytescore_similarity` requires both parts to map to CL ids above the match threshold. Compound pairs use the Hungarian path described above. +For simple pairs, `cytescore_similarity` requires both parts to map to CL ids above the match threshold. Compound pairs use the reducer described above. ## Workflow diagram (overview) ```mermaid flowchart TD - subgraph setup ["Setup (from_config)"] - H["Load CL CSV + OWL"] + subgraph setup ["Setup (setup.py + from_config)"] + H["Load CL CSV / enriched CSV + OWL"] I["LLM descriptions for CL terms"] J["Embed CL descriptions"] K["Save ontology JSON + NPZ"] @@ -102,13 +110,14 @@ flowchart TD subgraph compare ["Compare (per run_id)"] A["Input: author + algorithm labels"] + L0["Lowercase non-empty labels"] D0["LLM decompose mixture labels"] B["LLM descriptions for label parts"] C["Embed description text"] D["Match parts to CL via cosine similarity"] - E["Score per pair (simple or Hungarian compound)"] + E["Score per pair (simple or compound reducer)"] F["Results DataFrame"] - A --> D0 --> B --> C --> D --> E --> F + A --> L0 --> D0 --> B --> C --> D --> E --> F end setup --> compare @@ -122,7 +131,8 @@ flowchart TD ```mermaid flowchart TD - A["Author label + algorithm label"] --> B["Decompose each label into parts"] + A["Author label + algorithm label"] --> L["Lowercase"] + L --> B["Decompose each label into parts"] B --> C["Describe + embed each unique part"] C --> D["Match each part to nearest CL term"] @@ -133,20 +143,23 @@ flowchart TD G --> I["similarity_method: cytescore"] H --> J{"Which side matched?"} J -->|Neither| K["no_matches"] - J -->|One| L["partial_match"] + J -->|One| Lpm["partial_match"] - E -->|Yes| M["Hungarian match mean + coverage"] - M --> N["similarity_method: cytescore_compound"] + E -->|Yes| M{"compound_scoring"} + M -->|max| Nmax["max of score matrix S"] + M -->|hungarian_mean| Nhm["Hungarian mean + coverage"] + Nmax --> N["similarity_method: cytescore_compound"] + Nhm --> N I --> O["Result row"] K --> O - L --> O + Lpm --> O N --> O ``` ## AnnData entry point -`compare_anndata` reads algorithm columns from `adata.obs` and delegates to `compare` with the same `run_id` and caching semantics. Author labels are still passed explicitly as a list. +`compare_anndata` reads algorithm columns from `adata.obs` and delegates to `compare` with the same `run_id`, `compound_scoring`, and caching semantics. Author labels are still passed explicitly as a list and are lowercased the same way. ## Related documentation diff --git a/modal_app/README.md b/modal_app/README.md index 4e53bb4..8f95eb4 100644 --- a/modal_app/README.md +++ b/modal_app/README.md @@ -131,7 +131,7 @@ uv run modal run -m modal_app --env cytetrainer setup uv run modal run -m modal_app --env cytetrainer setup --force ``` -`setup` downloads the CL CSV, the OWL file, and the precomputed Kimi-K2.6 descriptions and qwen3-embedding-8b embeddings into the `cyteonto` Modal volume. It is idempotent; `--force` overwrites existing files. +`setup` downloads the CL CSV, the OWL file, the enriched CL CSV (or builds it locally if missing from the CDN), and the precomputed primary-pair descriptions and embeddings into the `cyteonto` Modal volume. Backup-pair artifacts are optional: if those downloads fail, setup continues with the primary pair only. It is idempotent; `--force` overwrites existing files. ## Custom domain @@ -196,6 +196,7 @@ Request body: | `metric` | `str` | no | `cosine_kernel` | See the `cyteonto` README for the full list. | | `metricParams` | `dict \| null` | no | `null` | Metric-specific parameters (for example `{"width": 0.25}` for `cosine_kernel`). | | `minMatchSimilarity` | `float in [0, 1]` | no | `0.1` | Threshold below which a label is considered unmatched to any CL term. | +| `compoundScoring` | `max \| hungarian_mean` | no | `max` | How to reduce the m×n part score matrix for compound pairs. `max` takes the highest entry; `hungarian_mean` uses Hungarian assignment mean with a coverage penalty when part counts differ. | | `useCache` | `bool` | no | `true` | If `false`, all on-disk caches are bypassed for this run. | Response: @@ -244,15 +245,15 @@ Returns `404` if the `runId` is unknown and `409` if the run is not in state `co | `run_id` | `str` | Echoed from the request. | | `algorithm` | `str` | Key from the `algorithms` map. | | `pair_index` | `int` | Position in the label list, starting at 0. | -| `author_label` | `str` | Author label for this pair. | -| `algorithm_label` | `str` | Algorithm label for this pair. | -| `author_ontology_id` | `str` | Best CL match for the author label. Semicolon-separated for compound pairs (matched assignment only). Empty string if unmatched. | +| `author_label` | `str` | Author label for this pair (lowercased at compare entry). | +| `algorithm_label` | `str` | Algorithm label for this pair (lowercased at compare entry). | +| `author_ontology_id` | `str` | Best CL match per author part, semicolon-separated in part order. Empty string if unmatched. | | `author_ontology_name` | `str` | Primary CSV label (or OWL fallback) for each id in `author_ontology_id`. | -| `author_embedding_similarity` | `float` | Mean cosine similarity of author parts to their CL matches. | -| `algorithm_ontology_id` | `str` | Best CL match for the algorithm label. Same compound rules as author. | +| `author_embedding_similarity` | `float \| str` | Per-part cosine similarity to the matched CL term. Single float when one part; semicolon-separated floats when multiple parts. | +| `algorithm_ontology_id` | `str` | Best CL match per algorithm part, same rules as author. | | `algorithm_ontology_name` | `str` | Names for ids in `algorithm_ontology_id`. | -| `algorithm_embedding_similarity` | `float` | Mean cosine similarity of algorithm parts to their CL matches. | -| `cytescore_similarity` | `float` | Score under the chosen `metric`; `0.0` when scoring does not apply. | +| `algorithm_embedding_similarity` | `float \| str` | Same shape as author embedding similarity, for algorithm parts. | +| `cytescore_similarity` | `float` | Score under the chosen `metric` / `compoundScoring` reducer; `0.0` when scoring does not apply. | | `similarity_method` | `str` | `cytescore`, `cytescore_compound`, `string_similarity`, `partial_match`, `no_matches`, or `empty`. | ### GET `/health` @@ -345,6 +346,7 @@ jq -n \ metric: "cosine_kernel", metricParams: { center: 1.0, width: 0.25, amplitude: 1.0 }, minMatchSimilarity: 0.15, + compoundScoring: "max", useCache: true }' \ | curl -sS -X POST "$CYTEONTO_URL/compare" \ @@ -418,6 +420,7 @@ Mounted at `/cyteonto_data` inside every container: /cyteonto_data/ ├── cell_ontology/ │ ├── cell_to_cell_ontology.csv +│ ├── cell_to_cell_ontology_enriched.csv │ └── cl.owl ├── embedding/ │ ├── cell_ontology/ @@ -442,7 +445,7 @@ Mounted at `/cyteonto_data` inside every container: └── decompositions_.json ``` -`` is the cleaned LLM model name (for example `moonshotai-Kimi-K2.6`) and `` is the cleaned embedding model name. Descriptions and embeddings are cached per text and embedding model, so switching either model produces a fresh cache without invalidating the existing one. +`` is the cleaned LLM model name (for example `moonshotai-Kimi-K2.6`) and `` is the cleaned embedding model name. Descriptions and embeddings are cached per text and embedding model, so switching either model produces a fresh cache without invalidating the existing one. The enriched CSV adds `label_normalized` for case-insensitive ontology lookups while preserving original synonym casing for display names. You can inspect the volume directly: diff --git a/tests/README.md b/tests/README.md index b4fc385..3b2639c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,11 +14,12 @@ tests/ ├── test_models.py # Pydantic models (descriptions, configs, usage) ├── test_path_config.py # PathConfig path resolution ├── test_artifact_keys.py # Artifact keys, paths, storage round trips -├── test_ontology.py # OntologyMapping + OntologySimilarity +├── test_ontology.py # OntologyMapping + OntologySimilarity (incl. label_normalized) +├── test_setup.py # setup.py enriched CSV + optional backup downloads ├── test_embed.py # Embedding HTTP helpers and orchestration ├── test_describe.py # Prompt building and error formatting ├── test_pubmed.py # PubMed abstract retrieval -└── test_cyteonto.py # Orchestrator units (decomposition, Hungarian compound scoring, compare) +└── test_cyteonto.py # Orchestrator units (decomposition, compound max/hungarian scoring, compare) ``` ## Running @@ -33,12 +34,15 @@ uv run pytest -v # Single test uv run pytest tests/test_models.py::TestCellDescription::test_to_sentence -# Compound scoring and compare integration +# Compound scoring and compare integration (max default + hungarian_mean) uv run pytest tests/test_cyteonto.py::TestHungarianMatchMean tests/test_cyteonto.py::TestCompareCompoundLabels -v # Label decomposition helpers (in test_cyteonto.py) uv run pytest tests/test_cyteonto.py -k "Decompos or decompose" -v +# Setup enriched CSV / optional backup behavior +uv run pytest tests/test_setup.py -v + # Drop into debugger on failure uv run pytest --pdb ```