From 74f83c00cd3a6a51a42685b1563acc553406471b Mon Sep 17 00:00:00 2001 From: Shenbo Xu <27264836+xushenbo@users.noreply.github.com> Date: Fri, 17 Apr 2026 08:33:51 -0400 Subject: [PATCH 1/2] Add source adapter layer for MarketScan, LCED, and CPRD - Three ETL adapters (marketscan, lced, cprd) producing canonical tables - Vocabulary loaders: NDC, RxNorm, LOINC, Read code, prodcode, ICD GEM/CCS stubs - to_ehrdata() bridge converting canonical tables to EHRData - 327 tests across all modules - Reference docs: docs/sources/{marketscan,lced,cprd}.md - Tutorial notebooks: CPRD overview, LOINC mapping, ICD mapping, LCED cohort --- CHANGELOG.md | 113 +- docs/api/io_index.md | 13 +- docs/index.md | 8 + docs/sources/cprd.md | 110 ++ docs/sources/index.md | 12 + docs/sources/lced.md | 114 ++ docs/sources/marketscan.md | 79 ++ docs/tutorials/index.md | 29 +- docs/tutorials/source_cprd_overview.ipynb | 1242 +++++++++++++++++ docs/tutorials/source_icd_mapping.ipynb | 770 ++++++++++ docs/tutorials/source_lced_cohort.ipynb | 840 +++++++++++ docs/tutorials/source_loinc_mapping.ipynb | 806 +++++++++++ src/ehrdata/io/__init__.py | 2 +- src/ehrdata/io/source/__init__.py | 30 + src/ehrdata/io/source/adapters/__init__.py | 9 + src/ehrdata/io/source/adapters/cprd.py | 254 ++++ src/ehrdata/io/source/adapters/lced.py | 476 +++++++ src/ehrdata/io/source/adapters/marketscan.py | 353 +++++ src/ehrdata/io/source/extract.py | 182 +++ src/ehrdata/io/source/normalize.py | 221 +++ src/ehrdata/io/source/schema.py | 174 +++ src/ehrdata/io/source/to_ehrdata.py | 183 +++ src/ehrdata/io/source/vocab/__init__.py | 9 + src/ehrdata/io/source/vocab/icd.py | 86 ++ src/ehrdata/io/source/vocab/loinc.py | 67 + src/ehrdata/io/source/vocab/ndc.py | 68 + src/ehrdata/io/source/vocab/prodcode.py | 64 + src/ehrdata/io/source/vocab/readcode.py | 58 + src/ehrdata/io/source/vocab/rxnorm.py | 62 + tests/data/source_basic/diagnosis.csv | 6 + tests/data/source_basic/diagnosis_wide.csv | 4 + tests/data/source_basic/labtest.csv | 5 + tests/data/source_basic/procedure.csv | 5 + tests/data/source_basic/therapy.csv | 4 + tests/data/source_cprd/additional.tsv | 4 + tests/data/source_cprd/clinical.tsv | 6 + tests/data/source_cprd/patient.tsv | 4 + tests/data/source_cprd/practice.tsv | 3 + tests/data/source_cprd/referral.tsv | 3 + tests/data/source_cprd/test.tsv | 4 + tests/data/source_cprd/therapy.tsv | 6 + tests/data/source_vocab/loinc_map.csv | 11 + tests/data/source_vocab/medical_map.txt | 6 + .../data/source_vocab/ndc_ingredient_map.txt | 11 + tests/data/source_vocab/product_map.txt | 4 + .../source_vocab/rxcui_ingredient_map.txt | 11 + tests/io/test_source_cprd.py | 474 +++++++ tests/io/test_source_extract.py | 204 +++ tests/io/test_source_lced.py | 550 ++++++++ tests/io/test_source_marketscan.py | 485 +++++++ tests/io/test_source_normalize.py | 334 +++++ tests/io/test_source_schema.py | 116 ++ tests/io/test_source_to_ehrdata.py | 311 +++++ tests/io/test_source_vocab.py | 157 +++ 54 files changed, 9031 insertions(+), 131 deletions(-) create mode 100644 docs/sources/cprd.md create mode 100644 docs/sources/index.md create mode 100644 docs/sources/lced.md create mode 100644 docs/sources/marketscan.md create mode 100644 docs/tutorials/source_cprd_overview.ipynb create mode 100644 docs/tutorials/source_icd_mapping.ipynb create mode 100644 docs/tutorials/source_lced_cohort.ipynb create mode 100644 docs/tutorials/source_loinc_mapping.ipynb create mode 100644 src/ehrdata/io/source/__init__.py create mode 100644 src/ehrdata/io/source/adapters/__init__.py create mode 100644 src/ehrdata/io/source/adapters/cprd.py create mode 100644 src/ehrdata/io/source/adapters/lced.py create mode 100644 src/ehrdata/io/source/adapters/marketscan.py create mode 100644 src/ehrdata/io/source/extract.py create mode 100644 src/ehrdata/io/source/normalize.py create mode 100644 src/ehrdata/io/source/schema.py create mode 100644 src/ehrdata/io/source/to_ehrdata.py create mode 100644 src/ehrdata/io/source/vocab/__init__.py create mode 100644 src/ehrdata/io/source/vocab/icd.py create mode 100644 src/ehrdata/io/source/vocab/loinc.py create mode 100644 src/ehrdata/io/source/vocab/ndc.py create mode 100644 src/ehrdata/io/source/vocab/prodcode.py create mode 100644 src/ehrdata/io/source/vocab/readcode.py create mode 100644 src/ehrdata/io/source/vocab/rxnorm.py create mode 100644 tests/data/source_basic/diagnosis.csv create mode 100644 tests/data/source_basic/diagnosis_wide.csv create mode 100644 tests/data/source_basic/labtest.csv create mode 100644 tests/data/source_basic/procedure.csv create mode 100644 tests/data/source_basic/therapy.csv create mode 100644 tests/data/source_cprd/additional.tsv create mode 100644 tests/data/source_cprd/clinical.tsv create mode 100644 tests/data/source_cprd/patient.tsv create mode 100644 tests/data/source_cprd/practice.tsv create mode 100644 tests/data/source_cprd/referral.tsv create mode 100644 tests/data/source_cprd/test.tsv create mode 100644 tests/data/source_cprd/therapy.tsv create mode 100644 tests/data/source_vocab/loinc_map.csv create mode 100644 tests/data/source_vocab/medical_map.txt create mode 100644 tests/data/source_vocab/ndc_ingredient_map.txt create mode 100644 tests/data/source_vocab/product_map.txt create mode 100644 tests/data/source_vocab/rxcui_ingredient_map.txt create mode 100644 tests/io/test_source_cprd.py create mode 100644 tests/io/test_source_extract.py create mode 100644 tests/io/test_source_lced.py create mode 100644 tests/io/test_source_marketscan.py create mode 100644 tests/io/test_source_normalize.py create mode 100644 tests/io/test_source_schema.py create mode 100644 tests/io/test_source_to_ehrdata.py create mode 100644 tests/io/test_source_vocab.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fe1f53c..25d0da44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,113 +8,16 @@ and this project adheres to [Semantic Versioning][]. [keep a changelog]: https://keepachangelog.com/en/1.0.0/ [semantic versioning]: https://semver.org/spec/v2.0.0.html -## [0.2.0] - -### Fixed - - Assigning `.X` to a view of an X-less {class}`~ehrdata.EHRData` (e.g. one created with `layers=` only) no longer raises `TypeError: 'NoneType' object does not support item assignment`. The view is now materialised before the assignment, consistent with how AnnData handles other field modifications on views. ([#233](https://github.com/theislab/ehrdata/pull/233)) @eroell - -### Modified - - {func}`~ehrdata.infer_feature_types` considers integers from 0, ..., n as numeric. It further provides a new argument `binary_as`, to steer if columns 0/1 should be considered numeric or categorical. ([#231](https://github.com/theislab/ehrdata/pull/231)) @eroell - -## [0.1.2] - -### Added - - {func}`~ehrdata.io.from_pandas` with `format='long'` provides a new keyword argument `fill_time_gaps` that fills missing timegaps in the common case of integer time steps from 0 to n_timesteps ([#229](https://github.com/theislab/ehrdata/pull/229)) @eroell - -### Modified - - {func}`~ehrdata.dt.mimic_2` column `censor_flg` switched to lifeline's convention with 1=event, 0=censored, before this dataset loader function had them vice versa since the dataset provides them as such originally. ([#227](https://github.com/theislab/ehrdata/pull/227)) @sueoglu - -### Fixed - - {func}`~ehrdata.io.from_pandas` with `format='long'` misordered entries in `.X`/`.layers` with `.obs` if the input df was not sorted for the obs id keys, which is now fixed. ([#228](https://github.com/theislab/ehrdata/pull/228)) @eroell - -### Documentation - - Documentation style polishing ([#223](https://github.com/theislab/ehrdata/pull/223)) @zethson - -## [0.1.1] - -### Added - - {func}`~ehrdata.io.omop.setup_connection` can read `.parquet` files. ([#217](https://github.com/theislab/ehrdata/pull/217)) @eroell - -### Fixed - - Sliceing of `EHRData` objects fixed when the backing object is an `AnnData`. ([#218](https://github.com/theislab/ehrdata/pull/218)) @eroell - -### Maintenance - - More concise messages in {func}`~ehrdata.infer_feature_types`. ([#215](https://github.com/theislab/ehrdata/pull/215)) @zethson - - -## [0.1.0] +## [Unreleased] ### Added - - {func}`~ehrdata.move_to_obs` and {func}`~ehrdata.move_to_x` are new helpers for conveniently moving variables from central 2D arrays to the `.obs` field, and vice versa. ([#199](https://github.com/theislab/ehrdata/pull/201)) @eroell - - {func}`~ehrdata.dt.physionet2019` as another out-of-the-box, conveniently available dataset with 40'000 ICU stays from the Physionet 2019 challenge. ([#204](https://github.com/theislab/ehrdata/pull/204)) @eroell - - `time_precision` parameter (`"date"` or `"datetime"`) to {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` for finer temporal granularity control. ([#210](https://github.com/theislab/ehrdata/pull/210)) @eroell - -### Fixed -- {func}`~ehrdata.io.read_h5ad` fixed issues when `backed=True`. ([#199](https://github.com/theislab/ehrdata/pull/199)) @eroell -- {func}`~ehrdata.io.read_h5ad` fixed bug when `.X` is `None` and `harmonize_missing_features` is `True`. ([#206](https://github.com/theislab/ehrdata/pull/206)) @eroell -- {func}`~ehrdata.io.omop.setup_obs` with `observation_table="person_visit_occurrence"` now supports multiple visits per patient, creating one row per visit with unique observation IDs, instead of failing with xarray conversion errors with non-unique indices. ([#210](https://github.com/theislab/ehrdata/pull/210)) @eroell -- OMOP time interval boundaries now use half-open intervals `[start, end)` to prevent duplicate measurements at interval boundaries. ([#210](https://github.com/theislab/ehrdata/pull/210)) @eroell - -### Maintenance -- Support Python3.14 ([#194](https://github.com/theislab/ehrdata/pull/194)) @Zethson -- Address `FutureWarning`s across multiple places ([#200](https://github.com/theislab/ehrdata/pull/200)) @eroell -- Enhanced tutorial structure ([#208](https://github.com/theislab/ehrdata/pull/208)) @eroell - -### Modified -- Dataset generator function `ed.dt.ehrdata_blobs` now takes `n_cat_var` and `n_categories` arguments to generate categorical (integer encoded) time series data ([#207](https://github.com/theislab/ehrdata/pull/207)) @sueoglu -- If `enrich_var_with_feature_info=True` in {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables`, `data_table_concept_ids` not included within the concept table are now mapped from their respective alternate `concept_id` included in the concept_relationship table to retrieve the available feature information. ([#205](https://github.com/theislab/ehrdata/pull/205)) @KilianDahm -- {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` with use of `"person"` now checks `birth_datetime` for meaningful behaviour and error messages. ([#210](https://github.com/theislab/ehrdata/pull/210)) @eroell -- {func}`~ehrdata.integrations.vitessce.gen_default_config` provides convenience to generate a config directly from an `EHRData` object, and should be used instead of the previous `ehrdata.integrations.vitessce.gen_config`. ([#211](https://github.com/theislab/ehrdata/pull/211)) @eroell - -## [0.0.10] - -{class}`~ehrdata.EHRData` drops the `.R` field, and now supports 3D data storage in any slot of `.layers`. See the {doc}`tutorials/getting_started` tutorial for an introduction to this behaviour. In the future, `.X` will be enabled soon for 3D data storage as well. - -### Maintenance -- Enhanced {doc}`tutorials/getting_started` ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- Move from zarr<3 to zarr>=3 ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell - -### Fixed - -### Modified -- `EHRData` drops the `.R` field in favor of using `.layers` for any 3D data arrays ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- `EHRData`'s shape property will always return a 3 dimensional shape. If an `EHRData` object has flat arrays only, the third dimension will be 1. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- The following functions now take a `layer` argument: {func}`~ehrdata.io.read_csv`, {func}`~ehrdata.io.from_pandas`, {func}`~ehrdata.io.to_pandas`, {func}`~ehrdata.io.omop.setup_variables`, {func}`~ehrdata.io.omop.setup_interval_variables`, {func}`~ehrdata.dt.ehrdata_blobs`, {func}`~ehrdata.dt.physionet2012`. If it is let to its default, `None`, the `.X` field of `EHRData` is used. Since `.X` is 2D in this release, in cases with 3D data, the `layer` argument needs to be used. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- {func}`~ehrdata.io.write_zarr` now writes an `EHRData` specific store encoding, with `anndata` as a substore. This change allows to use `AnnData` with its change to consolidated Zarr metadata, and better isolates `AnnData`'s io. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell -- {func}`~ehrdata.io.read_zarr` is adapted to read the new store encoding, and can also deal with `AnnData` stores. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell - - -## [0.0.9] - -### Maintenance -- Use custom logger & remove pydata sparse ([#176](https://github.com/theislab/ehrdata/pull/176)) @Zethson -- Replace figshare with scverse S3 ([#177](https://github.com/theislab/ehrdata/pull/177)) @Zethson -- Update template to v0.6.0 ([#166](https://github.com/theislab/ehrdata/pull/166)) @Zethson - -### Fixed -- Fix order of `var` created in `ed.io.omop.setup_variables` and `ed.io.omop.setup_interval_variables` ([#179](https://github.com/theislab/ehrdata/pull/179)) @eroell - -### Modified -- Rename `ed.pl.vitessce.gen_config` to `ed.integrations.vitessce.gen_config` ([#181](https://github.com/theislab/ehrdata/pull/181)) @eroell -- Rename `ed.tl.omop.EHRDataset` to `ed.integrations.torch.OMOPEHRDataset` ([#181](https://github.com/theislab/ehrdata/pull/181)) @eroell - - -## [0.0.8] - -### Fixed -- Update duckdb imports for future ([#157](https://github.com/theislab/ehrdata/pull/157)) @eroell - -### Maintenance -- Private subset method for `EHRData` ([#160](https://github.com/theislab/ehrdata/pull/160)) @eroell -- Remove `omop` package dependency ([#160](https://github.com/theislab/ehrdata/pull/160)) @eroell - -## [0.0.7] - -### Fixed -- Fix tests and Getting Started Notebook ([#155](https://github.com/theislab/ehrdata/pull/155)) @eroell - -### Maintenance -- Update duckdb imports for future ([#155](https://github.com/theislab/ehrdata/pull/155)) @eroell +- {func}`~ehrdata.io.source.to_ehrdata` converts canonical source tables (diagnosis, therapy, lab test, procedure) into an {class}`~ehrdata.EHRData` object +- `ehrdata.io.source.adapters.marketscan` — adapter for IBM MarketScan commercial claims (diagnosis, therapy, procedure, patinfo, insurance, provider) +- `ehrdata.io.source.adapters.lced` — adapter for IBM LCED linked claims-EMR data (diagnosis, therapy, lab test, habit, patinfo) +- `ehrdata.io.source.adapters.cprd` — adapter for CPRD GOLD UK primary-care data (diagnosis, therapy, lab test, patinfo); supports Read code and prodcode vocabulary translation +- `ehrdata.io.source.vocab` — vocabulary loaders for NDC→ingredient, RxCUI→ingredient, LOINC, Read code, and prodcode mappings +- Source reference documentation for MarketScan, LCED, and CPRD GOLD (`docs/sources/`) +- Tutorial notebooks: CPRD overview, LOINC mapping, ICD mapping, LCED cohort study (`docs/tutorials/`) ## [0.0.6] diff --git a/docs/api/io_index.md b/docs/api/io_index.md index 481ff46b..4946fd87 100644 --- a/docs/api/io_index.md +++ b/docs/api/io_index.md @@ -5,8 +5,6 @@ :no-index: ``` -## General I/O - ```{eval-rst} .. autosummary:: :toctree: io @@ -19,19 +17,10 @@ io.write_zarr io.from_pandas io.to_pandas - -``` - -## OMOP CDM - -```{eval-rst} -.. autosummary:: - :toctree: io - :nosignatures: - io.omop.setup_connection io.omop.setup_obs io.omop.setup_variables io.omop.setup_interval_variables + io.source.to_ehrdata ``` diff --git a/docs/index.md b/docs/index.md index f826fb2e..8fbabb46 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,6 +14,14 @@ references.md ``` +```{toctree} +:caption: 'Data Sources' +:hidden: true +:maxdepth: 2 + +sources/index +``` + ```{toctree} :caption: 'Gallery' :hidden: true diff --git a/docs/sources/cprd.md b/docs/sources/cprd.md new file mode 100644 index 00000000..e039cfbf --- /dev/null +++ b/docs/sources/cprd.md @@ -0,0 +1,110 @@ +# CPRD GOLD + +CPRD GOLD (Clinical Practice Research Datalink) is a UK primary-care database derived +from anonymised GP records. The extract described here is the May 2018 dementia cohort +(study DS059). The Python adapter lives at `ehrdata.io.source.adapters.cprd`. + +## Raw data + +**Format:** Zip archives of tab-delimited `.txt` files. Each file type (Clinical, +Referral, Test, Additional, Therapy, Consultation, Practice) is distributed as one +or more `.zip` files that are read directly without unpacking, using +`extract.read_zipped_tsvs`. + +**Patient identifier:** `patid` — renamed to `patient_id` by the adapter. + +**Date format:** `DD/MM/YYYY` — the adapter passes `formats=["%d/%m/%Y"]` to +`normalize.coerce_date`. + +**Scale (May 2018 extract):** + +| Category | Count | +|---|---| +| Total patients | 6,638,574 | +| Qualified patients | 6,613,198 | +| Total recordings | 3,726,733,860 | +| Total visits | 939,967,599 | + +**Coding systems:** + +| Domain | Column | Standard | +|---|---|---| +| Diagnosis | `medcode` | CPRD internal integer → Read code via `medical.txt` | +| Read code | `readcode` | Read code V2 (UK primary-care hierarchy) | +| Drug | `prodcode` | CPRD product dictionary → drug substance via `product.txt` | + +CPRD does **not** use ICD codes. The canonical `dxver` column is always `None` for +CPRD outputs. + +## Source files + +| File type | Key columns | Used for | +|---|---|---| +| Clinical | patid, eventdate, medcode, enttype, adid | diagnosis, labtest | +| Referral | patid, eventdate, medcode | diagnosis | +| Test | patid, eventdate, medcode, enttype, data1–data7 | diagnosis, labtest | +| Additional | patid, enttype, adid, data1–data7 | labtest (joined with Clinical) | +| Therapy | patid, eventdate, prodcode, bnfcode, qty, issueseq | therapy | +| Patient | patid, dobyr, sex, pracid | patinfo | +| Practice | pracid, region, lcd, uts | (reference; not in canonical output) | + +## Vocabulary files + +| File | Columns | Loaded by | +|---|---|---| +| `medical.txt` | medcode, readcode, desc | `vocab.readcode.load_medical_map` | +| `product.txt` | prodcode, drugsubstance, strength, … | `vocab.prodcode.load_product_map` | +| `product.csv` | prodcode, drugsubstance.updated, … | `vocab.prodcode.load_product_map` (preferred; uses `drugsubstance.updated`) | + +`product.csv` is the derived file produced by CPRD's Preparation step, which normalises +multi-ingredient names. When it is available, `load_product_map` uses +`drugsubstance.updated` automatically. + +## Canonical output tables + +| Canonical table | Sources | Notes | +|---|---|---| +| `diagnosis` | Clinical, Referral, Test | medcode translated to Read code when `medical_map` provided; `dxver = None` | +| `therapy` | Therapy | prodcode → drug substance when `product_map` provided; `prescription_date`, `start_date`, `end_date` = NaT; no NDC or RxCUI | +| `labtest` | Clinical+Additional (inner join on patid/adid/enttype) + Test | `data2`→value, `data3`→unit, `data4`→valuecat; `loinc = None` | +| `patinfo` | Patient | Three canonical columns only (patient_id, dobyr, sex) | + +## Lab test join + +CPRD stores lab values across two complementary file types: + +- **Additional** files hold the numeric values (`data1`–`data7`) keyed by `(patid, adid, enttype)` + but contain no `eventdate`. +- **Clinical** files hold `eventdate` keyed by the same composite key. + +`build_labtest` performs an inner join of Clinical and Additional on +`(patid, adid, enttype)` to recover `eventdate` for each data row, then unions with +Test files (which already contain both `eventdate` and data columns). + +## Usage + +```python +from ehrdata.io.source.adapters import cprd +from ehrdata.io.source.vocab import readcode, prodcode + +medical_map = readcode.load_medical_map("medical.txt") +product_map = prodcode.load_product_map("product.csv") # or product.txt + +diag = cprd.build_diagnosis(clinical, referral, test_data, medical_map=medical_map) +therapy = cprd.build_therapy(therapy_data, product_map=product_map) +labs = cprd.build_labtest(clinical, additional, test_data) +patinfo = cprd.build_patinfo(patient) +``` + +## Notes + +- Read codes starting with `R` (Symptoms, Signs and Ill-Defined Conditions) are not + disease codes and are typically excluded from phenotype analyses (filter + `~dx.str.startswith("R")`). +- Read codes starting with `ZZ` could not be mapped to any hierarchy in the original + extract and should be investigated before use. +- `issueseq` in the Therapy file is a repeat-prescription counter (1 = first issue, + 2 = first repeat, etc.). It is not mapped to the canonical `refill` column. +- The `practice` table provides region, last-collection date (`lcd`), and + up-to-standard date (`uts`) for each GP practice. These are useful for defining + study observation windows but are not included in the canonical `patinfo` output. diff --git a/docs/sources/index.md b/docs/sources/index.md new file mode 100644 index 00000000..7b3a100f --- /dev/null +++ b/docs/sources/index.md @@ -0,0 +1,12 @@ +# Data Sources + +Reference documentation for the three source datasets supported by the +`ehrdata.io.source` layer. + +```{toctree} +:maxdepth: 2 + +marketscan +lced +cprd +``` diff --git a/docs/sources/lced.md b/docs/sources/lced.md new file mode 100644 index 00000000..589f370d --- /dev/null +++ b/docs/sources/lced.md @@ -0,0 +1,114 @@ +# IBM LCED (Limited Claims-EMR Data) + +IBM LCED (Limited Claims-EMR Data) is a linked dataset that combines IBM MarketScan +claims with Explorys EMR data. It is a superset of MarketScan: every patient in +LCED also has claims records, and a subset additionally has structured EMR data from +the Explorys network. The Python adapter lives at +`ehrdata.io.source.adapters.lced`. + +## Raw data + +**Format:** PostgreSQL database, views prefixed with `v_`. + +**Patient identifier:** `patient_id` — already correct; no rename needed. + +**Scale (original extract):** + +| Table | Patients | Observations | +|---|---|---| +| v_drug | 2,614,657 | 132,831,286 | +| v_encounter | 3,630,723 | 191,489,491 | +| v_habit | 2,004,540 | 42,745,066 | +| v_observation | 2,952,436 | 1,249,354,089 | +| v_annual_summary_enrollment | 3,675,923 | 11,755,699 | +| v_detail_enrollment | 3,675,923 | 115,140,511 | +| v_facility_header | 2,496,039 | 127,987,056 | +| v_inpatient_admissions | 606,967 | 1,078,418 | +| v_inpatient_services | 606,967 | 38,790,918 | +| v_lab_results | 394,602 | 42,530,560 | +| v_outpatient_drug_claims | 2,682,183 | 143,163,508 | +| v_outpatient_services | 3,281,396 | 365,200,712 | +| **Total** | **4,367,831** | **3,057,791,040** | + +**Coding systems:** + +| Domain | Column | Standard | Source | +|---|---|---|---| +| Diagnosis | `dx1`–`dx9`+ | ICD-9-CM / ICD-10-CM | MarketScan views | +| Diagnosis version | `dxver` | `"9"` / `"0"`; frequently null | MarketScan views | +| Drug (EMR) | `rx_cui` | RxNorm CUI | v_drug (Explorys) | +| Drug (claims) | `ndcnum` | NDC-10 | v_outpatient_drug_claims | +| Lab (EMR) | `loinc_test_id` | LOINC | v_observation (Explorys) | +| Lab (claims) | `loinccd` | LOINC | v_lab_results | + +**Key data-source notes (from original ETL):** + +- Diagnosis information is only available in MarketScan views, not Explorys. +- `v_drug.rx_cui` has far fewer missing values than NDC codes; use it when available. +- `dxver` has extensive missingness; ICD-9 is inferred from E/V prefix by the adapter. +- `pdx` in inpatient tables is the same as `dx1`; it is not double-counted. +- `v_lab_results.resltcat` (categorical result) is only 3.1% non-missing and is preserved as `valuecat`. +- Habit data (`v_habit`) is Explorys-only and is not available in plain MarketScan. +- `v_encounter.encounter_date` is always null; only `encounter_join_id` is used to join habits. + +## Source tables and adapter mapping + +### Diagnosis + +| Source view | Key columns | +|---|---| +| v_facility_header | patient_id, svcdate, dx1–dx9, dxver | +| v_inpatient_admissions | patient_id, admdate, pdx, dx1–dx15, dxver | +| v_inpatient_services | patient_id, svcdate, pdx, dx1–dx4, dxver | +| v_lab_results | patient_id, svcdate, dx1, dxver | +| v_outpatient_services | patient_id, svcdate, dx1–dx4, dxver | + +Wide dx columns are unnested to one row per code; `dxver` backfilled for E/V-prefix codes. + +### Therapy + +Two sources are joined separately before union because they use different drug coding systems: + +| Source view | Key columns | Drug identifier | +|---|---|---| +| v_drug (Explorys) | patient_id, prescription_date, start_date, end_date, rx_cui | RxNorm → ingredient via `vocab/rxnorm.py` | +| v_outpatient_drug_claims (MarketScan) | patient_id, svcdate, daysupp, refill, ndcnum | NDC-11 → ingredient via `vocab/ndc.py` | + +### Lab test + +| Source view | Key columns | Mapping | +|---|---|---| +| v_observation (Explorys) | patient_id, observation_date, std_value, std_uom, loinc_test_id | → eventdate, value, unit, loinc | +| v_lab_results (MarketScan) | patient_id, svcdate, result, resltcat, resunit, loinccd | → eventdate, value, valuecat, unit, loinc | + +### Habit + +`v_habit` (patient_id, mapped_question_answer, encounter_join_id) is LEFT JOINed with +`v_encounter` (patient_id, encounter_date, encounter_join_id) to recover the event date. +Rows with null `mapped_question_answer` are dropped. + +### Other tables + +`patinfo`, `insurance`, and `provider` follow the same pattern as MarketScan. + +## Usage + +```python +from ehrdata.io.source.adapters import lced +from ehrdata.io.source.vocab import ndc, rxnorm, loinc + +ndc_map = ndc.load_ndc_ingredient_map("ndc_ingredient_map.txt") +rxcui_map = rxnorm.load_rxcui_ingredient_map("rxcui_ingredient_map.txt") +loinc_map = loinc.load_loinc_map("loinc_map.csv") + +diag = lced.build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, + lab_results, outpatient_services, +) +therapy = lced.build_therapy( + v_drug, v_outpatient_drug_claims, ndc_map=ndc_map, rxcui_map=rxcui_map +) +labs = lced.build_labtest(v_observation, v_lab_results, loinc_map=loinc_map) +habit = lced.build_habit(v_habit, v_encounter) +patinfo = lced.build_patinfo(v_annual_summary_enrollment) +``` diff --git a/docs/sources/marketscan.md b/docs/sources/marketscan.md new file mode 100644 index 00000000..41454e60 --- /dev/null +++ b/docs/sources/marketscan.md @@ -0,0 +1,79 @@ +# IBM MarketScan + +IBM MarketScan is a US administrative claims database covering commercially insured +and Medicare-eligible populations. The Python adapter lives at +`ehrdata.io.source.adapters.marketscan`. + +## Raw data + +**Format:** PostgreSQL database (schema `commercial`). Each table is a view over +the underlying claims warehouse. + +**Patient identifier:** `enrolid` (bigint) — renamed to `patient_id` by the adapter. + +**Date columns:** standard `DATE` type; no format conversion needed. + +**Coding systems:** + +| Domain | Column | Standard | +|---|---|---| +| Diagnosis | `dx1`–`dx15` | ICD-9-CM / ICD-10-CM | +| Diagnosis version | `dxver` | `"9"` = ICD-9, `"0"` = ICD-10; often null for ICD-9 E/V codes | +| Drug | `ndcnum` | NDC-10 (zero-padded to NDC-11 by the adapter) | +| Procedure | `proc1`–`proc15` | CPT, HCPCS, ICD procedure codes | + +## Source tables + +| Table | Key columns | Used for | +|---|---|---| +| `facility_header` | enrolid, dxver, svcdate, dx1–dx9, proc1–proc6, cob, coins, copay | diagnosis, procedure, insurance | +| `inpatient_admissions` | enrolid, dxver, admdate, pdx, dx1–dx15, pproc, proc1–proc15 | diagnosis, procedure | +| `inpatient_services` | enrolid, dxver, svcdate, pdx, dx1–dx4, proctyp, pdx, proc1, cob, coins, copay | diagnosis, procedure, insurance | +| `outpatient_services` | enrolid, dxver, svcdate, dx1–dx4, proctyp, proc1, cob, coins, copay | diagnosis, procedure, insurance | +| `outpatient_prescription_drugs` | enrolid, svcdate, daysupp, refill, ndcnum, cob, coins, copay | therapy, insurance | +| `enrollment_annual_summary` | enrolid, dobyr, sex, efamid, year, region, … | patinfo | +| `enrollment_detail` | enrolid, dobyr, sex, dtstart, dtend, plantyp, rx, hlthplan, … | patinfo, provider | + +## Canonical output tables + +| Canonical table | Sources | Notes | +|---|---|---| +| `diagnosis` | facility_header, inpatient_admissions, inpatient_services, outpatient_services | Wide dx columns unnested to one row per code; ICD-9 inferred for E/V-prefix codes with null `dxver` | +| `therapy` | outpatient_prescription_drugs | `ndcnum` zero-padded to `ndc11`; `end_date = fill_date + daysupp`; optional NDC→ingredient join | +| `procedure` | facility_header, inpatient_admissions, inpatient_services, outpatient_services | `inpatient_services` unnests `[pdx, proc1]` only, matching the original ETL | +| `patinfo` | enrollment_annual_summary + 6 other tables | Extra MarketScan columns (efamid, year, region, …) preserved when present across all inputs | +| `insurance` | facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services | cob / coins / copay coerced to float64 | +| `provider` | enrollment_detail | DISTINCT on dtstart, dtend, plantyp, rx, hlthplan | + +## Usage + +```python +import pandas as pd +from ehrdata.io.source.adapters import marketscan +from ehrdata.io.source.vocab import ndc + +# Load vocab (optional) +ndc_map = ndc.load_ndc_ingredient_map("path/to/ndc_ingredient_map.txt") + +# Build canonical tables +diag = marketscan.build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, outpatient_services +) +therapy = marketscan.build_therapy(outpatient_prescription_drugs, ndc_map=ndc_map) +proc = marketscan.build_procedure( + facility_header, inpatient_admissions, inpatient_services, outpatient_services +) +patinfo = marketscan.build_patinfo( + enrollment_annual_summary, enrollment_detail, + facility_header, inpatient_admissions, + inpatient_services, outpatient_prescription_drugs, outpatient_services, +) +``` + +## Notes + +- MarketScan does not include EMR lifestyle data (`habit` table is absent; use LCED for that). +- `dxver` is frequently null for ICD-9 codes that begin with `E` or `V`. + The adapter calls `normalize.infer_icd_version` to backfill these automatically. +- `ndcnum` in the raw table is NDC-10 (may be fewer than 11 digits). + The adapter zero-pads it to NDC-11 before any ingredient join. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index aaea67cf..e4bc5619 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -7,22 +7,25 @@ orphan: true The easiest way to get familiar with ehrdata is to follow along with our tutorials. Many are also designed to work seamlessly in Binder, a free cloud computing platform. -**The notebooks can be followed in this order:** - -1. **[Getting Started](getting_started)** - Learn the basics of the `EHRData` data structure -2. **[Real Dataset Example: PhysioNet 2019](real_dataset_example_physionet2019)** - Work with a public clinical dataset -3. **[OMOP Introduction](omop_intro)** - Learn how to read OMOP data into `EHRData` -4. **[OMOP Machine Learning](omop_ml)** - Quickstart to ML workflows based on an OMOP dataset -5. **[Interactive Visualization](interactive_visualization)** - Explore your data interactively with Vitessce -6. **[Advanced: Data Management with LaminDB](lamindb)** - Store, share, and track datasets in the cloud +## Quick start ```{toctree} -:maxdepth: 2 +:maxdepth: 3 getting_started -real_dataset_example_physionet2019 -omop_intro +omop_tables_tutorial +tutorial_time_series_with_pypots omop_ml -interactive_visualization -lamindb +physionet2012 +``` + +## Data Sources + +```{toctree} +:maxdepth: 3 + +source_cprd_overview +source_loinc_mapping +source_icd_mapping +source_lced_cohort ``` diff --git a/docs/tutorials/source_cprd_overview.ipynb b/docs/tutorials/source_cprd_overview.ipynb new file mode 100644 index 00000000..2ba018e7 --- /dev/null +++ b/docs/tutorials/source_cprd_overview.ipynb @@ -0,0 +1,1242 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c3d4", + "metadata": {}, + "source": [ + "# CPRD GOLD: Source Overview and Exploratory Data Analysis\n", + "\n", + "**CPRD GOLD** (Clinical Practice Research Datalink) is a UK primary-care database derived from anonymised GP records. This notebook walks through the canonical Python workflow for CPRD data using `ehrdata.io.source.adapters.cprd` — the direct Python translation of the original R EDA pipeline.\n", + "\n", + "**Topics covered**\n", + "1. Loading raw CPRD zip archives into DataFrames\n", + "2. Patient and event-date EDA across file types\n", + "3. Read code hierarchy analysis (diagnosis)\n", + "4. Drug-substance therapy summary\n", + "5. Qualified-patient filtering rules" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b1c2d3e4", + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from ehrdata.io.source.adapters import cprd\n", + "from ehrdata.io.source.vocab import readcode, prodcode" + ] + }, + { + "cell_type": "markdown", + "id": "c1d2e3f4", + "metadata": {}, + "source": [ + "## 1. Loading Raw CPRD Data\n", + "\n", + "CPRD distributes data as zip archives of tab-delimited `.txt` files. Each file type (Clinical, Referral, Test, Additional, Therapy, Patient) may span multiple `.zip` files.\n", + "\n", + "The `extract.read_zipped_tsvs` helper reads them directly without unpacking. Below we build synthetic DataFrames that mirror the real CPRD schema so you can follow along without access to raw data." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d1e2f3a4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Clinical rows : 5\n", + "Referral rows : 2\n", + "Test rows : 3\n", + "Additional rows: 3\n", + "Therapy rows : 4\n", + "Patient rows : 3\n" + ] + } + ], + "source": [ + "# --- Synthetic CPRD DataFrames (mirrors real schema) ---\n", + "\n", + "clinical = pd.DataFrame({\n", + " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\"],\n", + " \"eventdate\": [\"01/06/2010\", \"15/08/2012\", \"03/03/2008\", \"22/11/2015\", \"22/11/2015\"],\n", + " \"medcode\": [100, 200, 100, 300, 300],\n", + " \"enttype\": [4, 4, 4, 5, 5],\n", + " \"adid\": [\"A1\", \"A2\", \"A3\", \"A4\", \"A4\"],\n", + "})\n", + "\n", + "referral = pd.DataFrame({\n", + " \"patid\": [\"P002\", \"P003\"],\n", + " \"eventdate\": [\"14/07/2009\", \"01/01/2016\"],\n", + " \"medcode\": [200, 400],\n", + " \"enttype\": [3, 3],\n", + " \"adid\": [\"\", \"\"],\n", + "})\n", + "\n", + "test_data = pd.DataFrame({\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"eventdate\": [\"10/05/2011\", \"20/09/2013\", \"01/04/2017\"],\n", + " \"medcode\": [500, 100, 200],\n", + " \"enttype\": [4, 5, 4],\n", + " \"adid\": [\"A5\", \"A6\", \"A7\"],\n", + " \"data1\": [1.0, 2.0, 3.0], \"data2\": [7.1, 6.4, 8.9],\n", + " \"data3\": [\"%\", \"%\", \"%\"], \"data4\": [\"\", \"\", \"\"],\n", + " \"data5\": [None, None, None], \"data6\": [None, None, None], \"data7\": [None, None, None],\n", + "})\n", + "\n", + "additional = pd.DataFrame({\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"enttype\": [4, 4, 5],\n", + " \"adid\": [\"A1\", \"A3\", \"A4\"],\n", + " \"data1\": [1.0, 1.0, 2.0], \"data2\": [7.1, 6.4, 8.9],\n", + " \"data3\": [\"%\", \"%\", \"%\"], \"data4\": [\"\", \"\", \"\"],\n", + " \"data5\": [None, None, None], \"data6\": [None, None, None], \"data7\": [None, None, None],\n", + "})\n", + "\n", + "therapy_data = pd.DataFrame({\n", + " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\"],\n", + " \"eventdate\": [\"10/01/2010\", \"20/03/2011\", \"05/06/2009\", \"12/12/2014\"],\n", + " \"prodcode\": [\"PROD1\", \"PROD1\", \"PROD2\", \"PROD3\"],\n", + " \"bnfcode\": [\"6.1.2\", \"6.1.2\", \"6.1.1\", \"2.12\"],\n", + " \"qty\": [28, 28, 56, 28],\n", + " \"issueseq\": [1, 2, 1, 1],\n", + "})\n", + "\n", + "patient = pd.DataFrame({\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"yob\": [155, 162, 178], # encoded as year_of_birth - 1800\n", + " \"sex\": [\"Female\", \"Male\", \"Female\"],\n", + " \"pracid\": [\"PR1\", \"PR1\", \"PR2\"],\n", + " \"gender\": [2, 1, 2],\n", + " \"frd\": [\"01/01/2000\", \"01/06/1998\", \"15/03/2010\"],\n", + " \"tod\": [None, None, None],\n", + " \"deathdate\": [None, None, None],\n", + "})\n", + "\n", + "print(f\"Clinical rows : {len(clinical)}\")\n", + "print(f\"Referral rows : {len(referral)}\")\n", + "print(f\"Test rows : {len(test_data)}\")\n", + "print(f\"Additional rows: {len(additional)}\")\n", + "print(f\"Therapy rows : {len(therapy_data)}\")\n", + "print(f\"Patient rows : {len(patient)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a3b4", + "metadata": {}, + "source": [ + "## 2. Vocabulary Maps\n", + "\n", + "CPRD uses two internal code systems that must be translated:\n", + "\n", + "| Code | Meaning | Vocab file |\n", + "|---|---|---|\n", + "| `medcode` | Integer → Read code (diagnosis) | `medical.txt` |\n", + "| `prodcode` | String → drug substance (therapy) | `product.txt` / `product.csv` |\n", + "\n", + "Use `readcode.load_medical_map` and `prodcode.load_product_map` to load them." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f1a2b3c4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Medical map (medcode → readcode):\n", + "medcode readcode\n", + " 100 A10..00\n", + " 200 C10..00\n", + " 300 E12..00\n", + "\n", + "Product map (prodcode → drugsubstance):\n", + "prodcode drugsubstance\n", + " PROD1 metformin\n", + " PROD2 insulin glargine\n", + " PROD3 atorvastatin\n" + ] + } + ], + "source": [ + "# Synthetic vocabulary maps (in practice, load from medical.txt / product.csv)\n", + "medical_map_df = pd.DataFrame({\n", + " \"medcode\": [100, 200, 300, 400, 500],\n", + " \"readcode\": [\"A10..00\", \"C10..00\", \"E12..00\", \"F45..00\", \"44J3.00\"],\n", + " \"desc\": [\"Cholera\", \"Diabetes mellitus\", \"Obesity\", \"Anxiety disorder\", \"HbA1c\"],\n", + "})\n", + "\n", + "product_map_df = pd.DataFrame({\n", + " \"prodcode\": [\"PROD1\", \"PROD2\", \"PROD3\"],\n", + " \"drugsubstance\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", + " \"drugsubstance.updated\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", + "})\n", + "\n", + "# Use the loader functions on in-memory CSV to demonstrate the API\n", + "import tempfile, os\n", + "\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".txt\", delete=False) as f:\n", + " medical_map_df.to_csv(f, sep=\"\\t\", index=False)\n", + " med_path = f.name\n", + "\n", + "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".csv\", delete=False) as f:\n", + " product_map_df.to_csv(f, index=False)\n", + " prod_path = f.name\n", + "\n", + "medical_map = readcode.load_medical_map(med_path)\n", + "product_map = prodcode.load_product_map(prod_path)\n", + "\n", + "os.unlink(med_path)\n", + "os.unlink(prod_path)\n", + "\n", + "print(\"Medical map (medcode → readcode):\")\n", + "print(medical_map.head(3).to_string(index=False))\n", + "print()\n", + "print(\"Product map (prodcode → drugsubstance):\")\n", + "print(product_map.head(3).to_string(index=False))" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c4d5", + "metadata": {}, + "source": [ + "## 3. Building Canonical Tables\n", + "\n", + "The adapter exposes four builder functions that normalise each domain into a standard schema. Pass optional vocab maps to translate internal codes." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b2c3d4e5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Diagnosis rows: 9\n", + "patient_id object\n", + "dxver object\n", + "eventdate datetime64[ns]\n", + "dx object\n", + "dtype: object\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_iddxvereventdatedx
0P001None2010-06-01A10..00
1P001None2011-05-1044J3.00
2P001None2012-08-15C10..00
3P002None2008-03-03A10..00
4P002None2009-07-14C10..00
\n", + "
" + ], + "text/plain": [ + " patient_id dxver eventdate dx\n", + "0 P001 None 2010-06-01 A10..00\n", + "1 P001 None 2011-05-10 44J3.00\n", + "2 P001 None 2012-08-15 C10..00\n", + "3 P002 None 2008-03-03 A10..00\n", + "4 P002 None 2009-07-14 C10..00" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Diagnosis: Clinical + Referral + Test → one row per code per patient per date\n", + "diag = cprd.build_diagnosis(clinical, referral, test_data, medical_map=medical_map)\n", + "print(f\"Diagnosis rows: {len(diag)}\")\n", + "print(diag.dtypes)\n", + "diag.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c2d3e4f5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Therapy rows: 4\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_idfill_dateingredientrxcuindc11
0P0012010-01-10metforminNoneNone
1P0012011-03-20metforminNoneNone
2P0022009-06-05insulin glargineNoneNone
3P0032014-12-12atorvastatinNoneNone
\n", + "
" + ], + "text/plain": [ + " patient_id fill_date ingredient rxcui ndc11\n", + "0 P001 2010-01-10 metformin None None\n", + "1 P001 2011-03-20 metformin None None\n", + "2 P002 2009-06-05 insulin glargine None None\n", + "3 P003 2014-12-12 atorvastatin None None" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Therapy: prodcode translated to drug substance\n", + "therapy = cprd.build_therapy(therapy_data, product_map=product_map)\n", + "print(f\"Therapy rows: {len(therapy)}\")\n", + "therapy[[\"patient_id\", \"fill_date\", \"ingredient\", \"rxcui\", \"ndc11\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d2e3f4a5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Lab test rows: 6\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_ideventdatevalueunitloinc
0P0012010-06-017.1%None
1P0012011-05-107.1%None
2P0022008-03-036.4%None
3P0022013-09-206.4%None
4P0032015-11-228.9%None
5P0032017-04-018.9%None
\n", + "
" + ], + "text/plain": [ + " patient_id eventdate value unit loinc\n", + "0 P001 2010-06-01 7.1 % None\n", + "1 P001 2011-05-10 7.1 % None\n", + "2 P002 2008-03-03 6.4 % None\n", + "3 P002 2013-09-20 6.4 % None\n", + "4 P003 2015-11-22 8.9 % None\n", + "5 P003 2017-04-01 8.9 % None" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Lab test: Clinical + Additional inner-joined on (patid, adid, enttype) → eventdate recovered\n", + "# Then unioned with Test files that already contain eventdate\n", + "labs = cprd.build_labtest(clinical, additional, test_data)\n", + "print(f\"Lab test rows: {len(labs)}\")\n", + "labs[[\"patient_id\", \"eventdate\", \"value\", \"unit\", \"loinc\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e2f3a4b5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_iddobyrsex
0P001155Female
1P002162Male
2P003178Female
\n", + "
" + ], + "text/plain": [ + " patient_id dobyr sex\n", + "0 P001 155 Female\n", + "1 P002 162 Male\n", + "2 P003 178 Female" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Patient info: three canonical columns only\n", + "patinfo = cprd.build_patinfo(patient)\n", + "patinfo" + ] + }, + { + "cell_type": "markdown", + "id": "f2a3b4c5", + "metadata": {}, + "source": [ + "## 4. Patient and Event-Date EDA\n", + "\n", + "The original R pipeline computed unique `(patid, eventdate)` pairs across all file types to characterise the temporal footprint of the extract. The Python equivalent uses `pd.concat` + `.drop_duplicates()`." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a3b4c5d6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total unique (patid, eventdate) pairs : 13\n", + "Unique patients : 3\n", + "Date range : 2008-03-03 → 2017-04-01\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
events
eventdate
20081
20092
20102
20112
20121
20131
20141
20151
20161
20171
\n", + "
" + ], + "text/plain": [ + " events\n", + "eventdate \n", + "2008 1\n", + "2009 2\n", + "2010 2\n", + "2011 2\n", + "2012 1\n", + "2013 1\n", + "2014 1\n", + "2015 1\n", + "2016 1\n", + "2017 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_DATE_FMT = \"%d/%m/%Y\"\n", + "\n", + "def _patid_dates(df, date_col=\"eventdate\"):\n", + " return (\n", + " df[[\"patid\", date_col]]\n", + " .rename(columns={date_col: \"eventdate\"})\n", + " .assign(eventdate=lambda d: pd.to_datetime(d[\"eventdate\"], format=_DATE_FMT, errors=\"coerce\"))\n", + " .drop_duplicates()\n", + " )\n", + "\n", + "all_events = pd.concat([\n", + " _patid_dates(clinical),\n", + " _patid_dates(referral),\n", + " _patid_dates(test_data),\n", + " _patid_dates(therapy_data),\n", + "], ignore_index=True).drop_duplicates()\n", + "\n", + "print(f\"Total unique (patid, eventdate) pairs : {len(all_events):,}\")\n", + "print(f\"Unique patients : {all_events['patid'].nunique():,}\")\n", + "print(f\"Date range : {all_events['eventdate'].min().date()} → {all_events['eventdate'].max().date()}\")\n", + "all_events.groupby(all_events[\"eventdate\"].dt.year).size().rename(\"events\").to_frame().head(10)" + ] + }, + { + "cell_type": "markdown", + "id": "b3c4d5e6", + "metadata": {}, + "source": [ + "## 5. Read Code Hierarchy Analysis\n", + "\n", + "CPRD Read codes follow a five-character hierarchy. The first character indicates the major chapter (A = Infectious diseases, C = Endocrine, E = Mental health, etc.). Truncating to the first two characters (e.g. `C10...`) gives the sub-chapter level.\n", + "\n", + "The R pipeline matched patients to pre-defined code lists at this truncated level." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c3d4e5f6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
obs_countpatient_count
readcode_reduced
A10...32
C10...33
44J...11
E12...11
F45...11
\n", + "
" + ], + "text/plain": [ + " obs_count patient_count\n", + "readcode_reduced \n", + "A10... 3 2\n", + "C10... 3 3\n", + "44J... 1 1\n", + "E12... 1 1\n", + "F45... 1 1" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Merge diagnosis with the medical map to get read codes\n", + "diag_coded = diag.copy()\n", + "\n", + "# Filter: exclude codes beginning with 'R' (symptoms) or 'ZZ' (unmapped)\n", + "diag_coded = diag_coded[~diag_coded[\"dx\"].str.startswith((\"R\", \"ZZ\"), na=True)]\n", + "\n", + "# Build a 3-character hierarchy stub e.g. \"C10...\"\n", + "diag_coded[\"readcode_reduced\"] = diag_coded[\"dx\"].str[:3] + \"...\"\n", + "\n", + "# Observation count per stub\n", + "hierarchy_obs = (\n", + " diag_coded.groupby(\"readcode_reduced\")\n", + " .size()\n", + " .rename(\"obs_count\")\n", + " .sort_values(ascending=False)\n", + ")\n", + "\n", + "# Patient count per stub\n", + "hierarchy_pat = (\n", + " diag_coded.groupby(\"readcode_reduced\")[\"patient_id\"]\n", + " .nunique()\n", + " .rename(\"patient_count\")\n", + " .sort_values(ascending=False)\n", + ")\n", + "\n", + "pd.concat([hierarchy_obs, hierarchy_pat], axis=1).head(10)" + ] + }, + { + "cell_type": "markdown", + "id": "d3e4f5a6", + "metadata": {}, + "source": [ + "## 6. Therapy Drug-Substance Summary\n", + "\n", + "The R pipeline grouped `prodcode` by `drugsubstance.updated` (from `product.csv`) and counted observations and unique patients per substance." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e3f4a5b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
obs_countpatient_count
ingredient
atorvastatin11
insulin glargine11
metformin21
\n", + "
" + ], + "text/plain": [ + " obs_count patient_count\n", + "ingredient \n", + "atorvastatin 1 1\n", + "insulin glargine 1 1\n", + "metformin 2 1" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "drug_summary = (\n", + " therapy\n", + " .groupby(\"ingredient\", dropna=False)\n", + " .agg(\n", + " obs_count=(\"patient_id\", \"count\"),\n", + " patient_count=(\"patient_id\", \"nunique\"),\n", + " )\n", + " .sort_values(\"patient_count\", ascending=False)\n", + ")\n", + "drug_summary" + ] + }, + { + "cell_type": "markdown", + "id": "f3a4b5c6", + "metadata": {}, + "source": [ + "## 7. Qualified-Patient Filtering\n", + "\n", + "The original CPRD study applied the following eligibility rules before any analysis. These map directly to pandas boolean masks.\n", + "\n", + "| Rule | Meaning |\n", + "|---|---|\n", + "| `frd ∈ [1918-01-01, 2018-12-31)` | Valid first-registration date |\n", + "| `frd ≤ tod` or `tod` is null | Not transferred out before registration |\n", + "| `tod ∈ [1918-01-01, 2018-12-31)` or null | Valid transfer-out date |\n", + "| `frd < deathdate` or `deathdate` is null | Died after registration |\n", + "| `dob < deathdate` or `deathdate` is null | Died after birth |\n", + "| `yob + 1800 ≥ 1887` | Born after 1887 (data quality) |\n", + "| `gender ≠ 3` | Known gender |\n", + "\n", + "> **Note on `yob`**: CPRD stores year of birth as `year – 1800`, so `yob = 155` corresponds to 1955." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a4b5c6d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total patients : 3\n", + "Qualified patients: 3\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patidyobgenderfrddob
0P00115522000-01-011955-01-01
1P00216211998-06-011962-01-01
2P00317822010-03-151978-01-01
\n", + "
" + ], + "text/plain": [ + " patid yob gender frd dob\n", + "0 P001 155 2 2000-01-01 1955-01-01\n", + "1 P002 162 1 1998-06-01 1962-01-01\n", + "2 P003 178 2 2010-03-15 1978-01-01" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_FMT = \"%d/%m/%Y\"\n", + "\n", + "qp = patient.copy()\n", + "qp[\"frd\"] = pd.to_datetime(qp[\"frd\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"tod\"] = pd.to_datetime(qp[\"tod\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"deathdate\"] = pd.to_datetime(qp[\"deathdate\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"dob\"] = pd.to_datetime((qp[\"yob\"] + 1800).astype(str) + \"-01-01\", errors=\"coerce\")\n", + "\n", + "frd_ok = qp[\"frd\"].between(\"1918-01-01\", \"2018-12-30\")\n", + "tod_order_ok = (qp[\"frd\"] <= qp[\"tod\"]) | qp[\"tod\"].isna()\n", + "tod_range_ok = qp[\"tod\"].between(\"1918-01-01\", \"2018-12-30\") | qp[\"tod\"].isna()\n", + "death_frd_ok = (qp[\"frd\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", + "death_dob_ok = (qp[\"dob\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", + "yob_ok = (qp[\"yob\"] + 1800) >= 1887\n", + "gender_ok = qp[\"gender\"] != 3\n", + "\n", + "qualified = qp[frd_ok & tod_order_ok & tod_range_ok & death_frd_ok & death_dob_ok & yob_ok & gender_ok].drop_duplicates(subset=\"patid\")\n", + "\n", + "print(f\"Total patients : {len(qp):,}\")\n", + "print(f\"Qualified patients: {len(qualified):,}\")\n", + "qualified[[\"patid\", \"yob\", \"gender\", \"frd\", \"dob\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "b4c5d6e7", + "metadata": {}, + "source": [ + "## 8. Converting to EHRData\n", + "\n", + "Once the canonical tables are built and patients are filtered, the final step is loading everything into an `EHRData` object for downstream analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c4d5e6f7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "EHRData object with n_obs × n_vars = 3 × 8\n", + " obs: 'dobyr', 'sex'\n", + " var: 'concept_source', 'concept_code'\n", + " uns: 'source_io_source', 'source_io_tables'\n", + " shape of .X: (3, 8)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from ehrdata.io.source import to_ehrdata\n", + "\n", + "# Restrict to qualified patients\n", + "qualified_ids = set(qualified[\"patid\"])\n", + "patinfo_q = patinfo[patinfo[\"patient_id\"].isin(qualified_ids)]\n", + "diag_q = diag[diag[\"patient_id\"].isin(qualified_ids)]\n", + "therapy_q = therapy[therapy[\"patient_id\"].isin(qualified_ids)]\n", + "\n", + "edata = to_ehrdata(\n", + " patinfo_q,\n", + " diagnosis=diag_q,\n", + " therapy=therapy_q,\n", + " source=\"cprd\",\n", + ")\n", + "edata" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "d4e5f6a7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Patients (obs):\n", + " dobyr sex\n", + "patient_id \n", + "P001 155 Female\n", + "P002 162 Male\n", + "P003 178 Female\n", + "\n", + "Concepts (var):\n", + " concept_source concept_code\n", + "concept \n", + "diagnosis:44J3.00 diagnosis 44J3.00\n", + "diagnosis:A10..00 diagnosis A10..00\n", + "diagnosis:C10..00 diagnosis C10..00\n", + "diagnosis:E12..00 diagnosis E12..00\n", + "diagnosis:F45..00 diagnosis F45..00\n", + "therapy:atorvastatin therapy atorvastatin\n", + "therapy:insulin glargine therapy insulin glargine\n", + "therapy:metformin therapy metformin\n", + "\n", + "Presence matrix X:\n", + "[[1. 1. 1. 0. 0. 0. 0. 1.]\n", + " [0. 1. 1. 0. 0. 0. 1. 0.]\n", + " [0. 0. 1. 1. 1. 1. 0. 0.]]\n" + ] + } + ], + "source": [ + "print(\"Patients (obs):\")\n", + "print(edata.obs)\n", + "print()\n", + "print(\"Concepts (var):\")\n", + "print(edata.var)\n", + "print()\n", + "print(\"Presence matrix X:\")\n", + "print(edata.X)" + ] + }, + { + "cell_type": "markdown", + "id": "e4f5a6b7", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Step | Key function |\n", + "|---|---|\n", + "| Diagnosis (Clinical + Referral + Test) | `cprd.build_diagnosis(..., medical_map=medical_map)` |\n", + "| Therapy (prodcode → drug substance) | `cprd.build_therapy(..., product_map=product_map)` |\n", + "| Lab tests (Clinical + Additional + Test) | `cprd.build_labtest(clinical, additional, test_data)` |\n", + "| Patient info | `cprd.build_patinfo(patient)` |\n", + "| Qualified-patient filter | pandas boolean masks on `frd`, `tod`, `yob`, `gender` |\n", + "| EHRData bridge | `to_ehrdata(patinfo, diagnosis=..., therapy=...)` |\n", + "\n", + "**Next steps**: See `source_loinc_mapping.ipynb` for LOINC-based lab filtering, or `source_icd_mapping.ipynb` for ICD code mapping workflows." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/source_icd_mapping.ipynb b/docs/tutorials/source_icd_mapping.ipynb new file mode 100644 index 00000000..929c59e8 --- /dev/null +++ b/docs/tutorials/source_icd_mapping.ipynb @@ -0,0 +1,770 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c302", + "metadata": {}, + "source": [ + "# ICD Code Mapping: Version Inference, GEM, and CCS\n", + "\n", + "Claims databases (MarketScan, LCED) carry ICD diagnosis codes in two versions:\n", + "\n", + "- **ICD-9-CM** — used in the US until September 2015\n", + "- **ICD-10-CM** — mandatory from October 2015 onwards\n", + "\n", + "This notebook covers:\n", + "1. Version inference from code structure\n", + "2. CMS General Equivalence Mappings (GEM) — ICD-9 ↔ ICD-10 translation\n", + "3. AHRQ Clinical Classifications Software (CCS) — grouping codes into clinical categories\n", + "4. Building disease phenotype code lists\n", + "\n", + "All patterns translate the original R `diagnosis code.R` workflow." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b1c2d302", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from ehrdata.io.source.adapters import lced\n", + "from ehrdata.io.source import normalize" + ] + }, + { + "cell_type": "markdown", + "id": "c1d2e302", + "metadata": {}, + "source": [ + "## 1. ICD Version Structure\n", + "\n", + "ICD-9 and ICD-10 codes have distinct syntactic signatures:\n", + "\n", + "| Version | Pattern | Examples |\n", + "|---|---|---|\n", + "| ICD-9-CM | 3–5 digits, optionally dotted; E/V prefix codes | `250.00`, `E11`, `V45.86` |\n", + "| ICD-10-CM | Letter + 2 digits + optional 1–4 chars | `E11.9`, `I25.10`, `Z79.4` |\n", + "\n", + "The `dxver` column in MarketScan / LCED records this as `'9'` or `'0'`, but is frequently null for ICD-9 E/V codes." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d1e2f302", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dxdxver
0250.009
1E11.90
2I25.100
3410.00None
4V45.86None
5428.0None
6Z79.40
7160.1None
8E11None
9250009
\n", + "
" + ], + "text/plain": [ + " dx dxver\n", + "0 250.00 9\n", + "1 E11.9 0\n", + "2 I25.10 0\n", + "3 410.00 None\n", + "4 V45.86 None\n", + "5 428.0 None\n", + "6 Z79.4 0\n", + "7 160.1 None\n", + "8 E11 None\n", + "9 25000 9" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Representative diagnosis codes from LCED / MarketScan\n", + "test_codes = pd.DataFrame({\n", + " \"dx\": [\"250.00\", \"E11.9\", \"I25.10\", \"410.00\", \"V45.86\",\n", + " \"428.0\", \"Z79.4\", \"160.1\", \"E11\", \"25000\"],\n", + " \"dxver\": [\"9\", \"0\", \"0\", None, None,\n", + " None, \"0\", None, None, \"9\"],\n", + "})\n", + "test_codes" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a302", + "metadata": {}, + "source": [ + "## 2. Version Inference\n", + "\n", + "`normalize.infer_icd_version` fills in null `dxver` values by inspecting the code pattern. ICD-9 E/V prefix codes — `E\\d+` and `V\\d+` — are the main source of missingness in MarketScan." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f1a2b302", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dxdxverdxver_inferred
0250.0099
1E11.900
2I25.1000
3410.00None9
4V45.86None9
5428.0None9
6Z79.400
7160.1None9
8E11None9
92500099
\n", + "
" + ], + "text/plain": [ + " dx dxver dxver_inferred\n", + "0 250.00 9 9\n", + "1 E11.9 0 0\n", + "2 I25.10 0 0\n", + "3 410.00 None 9\n", + "4 V45.86 None 9\n", + "5 428.0 None 9\n", + "6 Z79.4 0 0\n", + "7 160.1 None 9\n", + "8 E11 None 9\n", + "9 25000 9 9" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_ICD9_PATTERN = re.compile(r'^[VvEe]?\\d')\n", + "_ICD10_PATTERN = re.compile(r'^[A-Za-z]\\d')\n", + "\n", + "def infer_icd_version(dx, dxver):\n", + " \"\"\"Return '9' or '0'; fall back to regex if dxver is null.\"\"\"\n", + " if pd.notna(dxver) and dxver in (\"9\", \"0\"):\n", + " return dxver\n", + " if pd.isna(dx):\n", + " return None\n", + " code = str(dx).strip()\n", + " if _ICD9_PATTERN.match(code):\n", + " return \"9\"\n", + " if _ICD10_PATTERN.match(code):\n", + " return \"0\"\n", + " return None\n", + "\n", + "test_codes[\"dxver_inferred\"] = [\n", + " infer_icd_version(row.dx, row.dxver)\n", + " for _, row in test_codes.iterrows()\n", + "]\n", + "test_codes" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c402", + "metadata": {}, + "source": [ + "## 3. CMS General Equivalence Mappings (GEM)\n", + "\n", + "The CMS GEM files provide bidirectional approximate translations between ICD-9 and ICD-10. `ehrdata.io.source.vocab.icd.load_gem` is a stub that will load the CMS file once provided.\n", + "\n", + "```\n", + "2018_I10gem.txt — ICD-10-CM → ICD-9-CM\n", + "2018_I9gem.txt — ICD-9-CM → ICD-10-CM\n", + "```\n", + "\n", + "The original R pipeline used the `2018_I10gem.txt` to check which phenotype codes had equivalents in the data:\n", + "```r\n", + "icd.gem <- fread(\".../2018_I10gem.txt\", col.names=c(\"icd10\", \"icd9\"))\n", + "sum(diagnosis.df$dx %in% icd.gem$icd10)\n", + "```\n", + "\n", + "Below we show the equivalent pattern with a small inline GEM table." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b2c3d402", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Codes with GEM equivalent: 3 / 10\n", + "\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dxdxver_inferredicd9
0E11.9025000
1I25.10041401
2E119250
\n", + "
" + ], + "text/plain": [ + " dx dxver_inferred icd9\n", + "0 E11.9 0 25000\n", + "1 I25.10 0 41401\n", + "2 E11 9 250" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Minimal GEM table (ICD-10 → ICD-9)\n", + "gem_i10 = pd.DataFrame({\n", + " \"icd10\": [\"E119\", \"E11\", \"I2510\", \"I110\", \"I259\", \"I509\", \"G309\"],\n", + " \"icd9\": [\"25000\", \"250\", \"41401\", \"40210\", \"41400\", \"42800\", \"33100\"],\n", + " \"flags\": [\"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\"],\n", + "})\n", + "\n", + "# Strip dots from codes before lookup (LCED stores codes without dots)\n", + "test_codes[\"dx_nodot\"] = test_codes[\"dx\"].str.replace(\".\", \"\", regex=False)\n", + "\n", + "# ICD-10 codes with a GEM equivalent\n", + "mapped = test_codes[test_codes[\"dx_nodot\"].isin(gem_i10[\"icd10\"])]\n", + "print(f\"Codes with GEM equivalent: {len(mapped)} / {len(test_codes)}\")\n", + "print()\n", + "\n", + "# Look up the ICD-9 equivalent\n", + "result = mapped.merge(gem_i10[[\"icd10\", \"icd9\"]], left_on=\"dx_nodot\", right_on=\"icd10\", how=\"left\")\n", + "result[[\"dx\", \"dxver_inferred\", \"icd9\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e402", + "metadata": {}, + "source": [ + "## 4. AHRQ Clinical Classifications Software (CCS)\n", + "\n", + "CCS groups the ~70,000 ICD codes into ~500 clinically meaningful categories. This is useful for exploratory analyses (e.g. \"all cardiovascular disease\") without enumerating every ICD code.\n", + "\n", + "`ehrdata.io.source.vocab.icd.load_ccs_map` is a stub that will load the AHRQ CCS CSV once provided.\n", + "\n", + "### ICD-10-CM CCS category search (R pattern → Python)\n", + "\n", + "```r\n", + "# R:\n", + "grep(\"heart failure\", icd.ccs.disease, ignore.case=TRUE, value=TRUE)\n", + "```\n", + "```python\n", + "# Python equivalent:\n", + "ccs_df[\"category_desc\"].str.contains(\"heart failure\", case=False)\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d2e3f402", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'heart failure': ['CIR019']\n", + "'diabetes': ['END002']\n", + "'chronic kidney': ['GEN003']\n", + "'cancer': ['NEO011']\n", + "'coronary': ['CIR007']\n" + ] + } + ], + "source": [ + "# Minimal ICD-10-CM CCS table (mirrors DXCCSR_v2020-3.CSV structure)\n", + "ccs_icd10 = pd.DataFrame({\n", + " \"icd10\": [\"E119\", \"E118\", \"I509\", \"I2510\", \"I110\", \"G309\", \"N183\", \"J449\", \"C50.9\"],\n", + " \"icd10_desc\": [\"T2DM unspecified\", \"T2DM other\", \"Heart failure\",\n", + " \"Coronary artery disease\", \"Hypertensive HF\", \"Alzheimer disease\",\n", + " \"CKD stage 3\", \"COPD unspecified\", \"Breast cancer\"],\n", + " \"ccs_category\": [\"END002\", \"END002\", \"CIR019\", \"CIR007\", \"CIR007\",\n", + " \"NVS010\", \"GEN003\", \"RSP010\", \"NEO011\"],\n", + " \"ccs_desc\": [\n", + " \"Diabetes mellitus without complication\",\n", + " \"Diabetes mellitus without complication\",\n", + " \"Heart failure\",\n", + " \"Coronary atherosclerosis and other heart disease\",\n", + " \"Coronary atherosclerosis and other heart disease\",\n", + " \"Alzheimer disease and related disorders\",\n", + " \"Chronic kidney disease\",\n", + " \"Chronic obstructive pulmonary disease and bronchiectasis\",\n", + " \"Cancer of breast\",\n", + " ],\n", + "})\n", + "\n", + "def search_ccs(df, pattern, col=\"ccs_desc\"):\n", + " mask = df[col].str.contains(pattern, case=False, na=False)\n", + " return df[mask][[\"ccs_category\", \"ccs_desc\"]].drop_duplicates()\n", + "\n", + "for term in [\"heart failure\", \"diabetes\", \"chronic kidney\", \"cancer\", \"coronary\"]:\n", + " hits = search_ccs(ccs_icd10, term)\n", + " print(f\"'{term}': {hits['ccs_category'].tolist()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a402", + "metadata": {}, + "source": [ + "## 5. Building Disease Phenotype Code Lists\n", + "\n", + "The original R pipeline defined disease cohorts using explicit ICD code lists. Here we translate the most common ones — Type 2 Diabetes, AMI, Stroke, CHF — into Python dictionaries." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f2a3b402", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "T2D ICD-9 codes: ['250.0', '25000', '25002', '2501', '25010', '25012'] ...\n", + "T2D ICD-10 prefix: E11*\n" + ] + } + ], + "source": [ + "# ICD-9 codes for Type 2 Diabetes (from lced_template_analysis.R)\n", + "T2D_ICD9 = [\n", + " \"250.0\", \"25000\", \"25002\",\n", + " \"2501\", \"25010\", \"25012\",\n", + " \"2504\", \"25040\", \"25042\",\n", + " \"2505\", \"25050\", \"25052\",\n", + " \"2506\", \"25060\", \"25062\",\n", + " \"2507\", \"25070\", \"25072\",\n", + " \"2509\", \"25090\", \"25092\",\n", + "]\n", + "# ICD-10 prefix\n", + "T2D_ICD10_PREFIX = \"E11\"\n", + "\n", + "# AMI (Acute Myocardial Infarction)\n", + "AMI_ICD9 = [\"410\"] + [f\"41{x}\" for x in [\"00\",\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\"]]\n", + "AMI_ICD10_PREFIX = \"I21\"\n", + "\n", + "# Stroke\n", + "STROKE_ICD9_PREFIXES = [\"160\", \"161\", \"162\", \"163\", \"430\", \"431\", \"432\", \"433\", \"434\", \"435\"]\n", + "STROKE_ICD10_PREFIXES = [\"I60\", \"I61\", \"I62\", \"I63\", \"I64\"]\n", + "\n", + "# CHF (Congestive Heart Failure)\n", + "CHF_ICD9 = [\"4280\", \"428\"] + [f\"428{x}\" for x in range(10)]\n", + "CHF_ICD10_PREFIX = \"I50\"\n", + "\n", + "print(\"T2D ICD-9 codes:\", T2D_ICD9[:6], \"...\")\n", + "print(\"T2D ICD-10 prefix:\", T2D_ICD10_PREFIX + \"*\")" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c502", + "metadata": {}, + "source": [ + "## 6. Phenotype Matching in a Diagnosis Table\n", + "\n", + "Once you have a canonical `diagnosis` DataFrame from the LCED adapter, matching against these code lists is straightforward." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b3c4d502", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "patient_id eventdate dx dxver t2d ami stroke chf\n", + " P001 2014-03-01 E119 0 True False False False\n", + " P001 2015-11-15 I509 0 False False False True\n", + " P002 2016-07-20 25000 9 True False False False\n", + " P003 2013-09-05 I2510 0 False False False False\n", + " P003 2017-04-10 41001 9 False False False False\n", + " P004 2014-12-01 410 9 False True False False\n", + " P004 2018-02-28 E119 0 True False False False\n", + "\n", + "Patients with T2D:\n", + "['P001' 'P002' 'P004']\n" + ] + } + ], + "source": [ + "# Synthetic diagnosis table (mirrors lced.build_diagnosis output)\n", + "diagnosis = pd.DataFrame({\n", + " \"patient_id\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\", \"P004\", \"P004\"],\n", + " \"eventdate\": pd.to_datetime([\n", + " \"2014-03-01\", \"2015-11-15\", \"2016-07-20\",\n", + " \"2013-09-05\", \"2017-04-10\", \"2014-12-01\", \"2018-02-28\",\n", + " ]),\n", + " \"dx\": [\"E119\", \"I509\", \"25000\", \"I2510\", \"41001\", \"410\", \"E119\"],\n", + " \"dxver\": [\"0\", \"0\", \"9\", \"0\", \"9\", \"9\", \"0\"],\n", + "})\n", + "\n", + "def has_icd(dx_series, icd9_list=None, icd10_prefixes=None, icd9_prefixes=None):\n", + " \"\"\"Return a boolean mask for rows matching ICD-9 list or ICD-10 prefix.\"\"\"\n", + " mask = pd.Series(False, index=dx_series.index)\n", + " if icd9_list:\n", + " dx_nodot = dx_series.str.replace(\".\", \"\", regex=False)\n", + " mask |= dx_nodot.isin(icd9_list)\n", + " if icd10_prefixes:\n", + " if isinstance(icd10_prefixes, str):\n", + " icd10_prefixes = [icd10_prefixes]\n", + " for pfx in icd10_prefixes:\n", + " mask |= dx_series.str.startswith(pfx, na=False)\n", + " if icd9_prefixes:\n", + " for pfx in icd9_prefixes:\n", + " mask |= dx_series.str.startswith(pfx, na=False)\n", + " return mask\n", + "\n", + "diagnosis[\"t2d\"] = has_icd(diagnosis[\"dx\"], icd9_list=T2D_ICD9, icd10_prefixes=\"E11\")\n", + "diagnosis[\"ami\"] = has_icd(diagnosis[\"dx\"], icd9_list=AMI_ICD9, icd10_prefixes=\"I21\")\n", + "diagnosis[\"stroke\"] = has_icd(diagnosis[\"dx\"], icd9_prefixes=STROKE_ICD9_PREFIXES, icd10_prefixes=STROKE_ICD10_PREFIXES)\n", + "diagnosis[\"chf\"] = has_icd(diagnosis[\"dx\"], icd9_list=CHF_ICD9, icd10_prefixes=\"I50\")\n", + "\n", + "print(diagnosis.to_string(index=False))\n", + "print()\n", + "print(\"Patients with T2D:\")\n", + "print(diagnosis[diagnosis[\"t2d\"]][\"patient_id\"].unique())" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e502", + "metadata": {}, + "source": [ + "## 7. Cross-Version Cohort Definition\n", + "\n", + "Studies spanning the ICD-9→ICD-10 transition (October 2015) need to include both code versions. The canonical pattern:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d3e4f502", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "T2D cohort with index dates:\n", + " patient_id t2d_index_date\n", + "0 P001 2014-03-01\n", + "1 P002 2016-07-20\n", + "2 P004 2018-02-28\n", + "\n", + "ICD versions in T2D rows:\n", + "dx dxver\n", + "E119 0 2\n", + "25000 9 1\n" + ] + } + ], + "source": [ + "# Patients with ≥1 T2D code in any version\n", + "t2d_patients = (\n", + " diagnosis[diagnosis[\"t2d\"]]\n", + " .groupby(\"patient_id\")[\"eventdate\"]\n", + " .min() # index date = first T2D code\n", + " .rename(\"t2d_index_date\")\n", + " .reset_index()\n", + ")\n", + "print(\"T2D cohort with index dates:\")\n", + "print(t2d_patients)\n", + "\n", + "# Verify code versions present\n", + "print()\n", + "print(\"ICD versions in T2D rows:\")\n", + "print(diagnosis[diagnosis[\"t2d\"]][[\"dx\", \"dxver\"]].value_counts().to_string())" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a502", + "metadata": {}, + "source": [ + "## 8. Using the `vocab/icd` Stubs\\n\\nThe `ehrdata.io.source.vocab.icd` module provides two stub functions that will load external mapping files when available:\\n\\n```python\\nfrom ehrdata.io.source.vocab import icd\\n\\n# When CMS GEM file is available:\\ngem = icd.load_gem_map(\\\"2018_I10gem.txt\\\") # columns: source_code, target_code, flags\\n\\n# When AHRQ CCS file is available:\\nccs = icd.load_ccs_map(\\\"DXCCSR_v2020-3.CSV\\\") # columns: icd_code, ccs_category, ccs_description\\n```\\n\\nUntil those files are provided, use the pattern from Section 5 above (explicit code lists) which is fully reproducible without external dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "f3a4b502", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_gem_map: GEM loader requires external CMS GEM file; not yet implemented.\n", + "load_ccs_map: CCS loader requires external AHRQ CCS file; not yet implemented.\n" + ] + } + ], + "source": [ + "from ehrdata.io.source.vocab import icd\n", + "\n", + "# These will raise NotImplementedError until the external files are supplied\n", + "try:\n", + " gem = icd.load_gem_map(\"2018_I10gem.txt\")\n", + "except (NotImplementedError, FileNotFoundError) as e:\n", + " print(f\"load_gem_map: {e}\")\n", + "\n", + "try:\n", + " ccs = icd.load_ccs_map(\"DXCCSR_v2020-3.CSV\")\n", + "except (NotImplementedError, FileNotFoundError) as e:\n", + " print(f\"load_ccs_map: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c602", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Task | Approach |\n", + "|---|---|\n", + "| Version inference | `normalize.infer_icd_version` or regex on code structure |\n", + "| ICD-10 → ICD-9 translation | `icd.load_gem(path)` (requires CMS GEM file) |\n", + "| Clinical grouping | `icd.load_ccs_map(path)` (requires AHRQ CCS file) |\n", + "| Phenotype matching | `dx.isin(code_list)` or `dx.str.startswith(prefix)` |\n", + "| Cross-version cohort | combine ICD-9 list + ICD-10 prefix in one boolean mask |\n", + "\n", + "**Next steps**: See `source_lced_cohort.ipynb` for a full T2D second-line therapy cohort that applies these phenotype definitions." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/source_lced_cohort.ipynb b/docs/tutorials/source_lced_cohort.ipynb new file mode 100644 index 00000000..f337b1bd --- /dev/null +++ b/docs/tutorials/source_lced_cohort.ipynb @@ -0,0 +1,840 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c303", + "metadata": {}, + "source": [ + "# LCED Cohort Study: T2D Second-Line Therapy\n", + "\n", + "This notebook implements a complete cohort study using IBM LCED data, translated from the original R `lced_template_analysis.R`. The study examines **second-line antidiabetic therapy initiation** in patients with Type 2 Diabetes (T2D) who have poor glycaemic control on metformin.\n", + "\n", + "## Study Design\n", + "\n", + "| Step | Definition |\n", + "|---|---|\n", + "| **Eligible population** | Adults (≥ 45 years) with ≥ 1 T2D ICD code (ICD-9: 250.x; ICD-10: E11.x) |\n", + "| **Index date** | First metformin prescription date |\n", + "| **Poor glycaemic control** | HbA1c ≥ 7 % (LOINC 4548-4) after metformin start, with HbA1c ≥ 9 % **or** two measurements ≤ 183 days apart |\n", + "| **Exposure** | First second-line drug (GLP-1, SGLT-2, DPP-4, or sulfonylurea) within 365 days of poor-control HbA1c |\n", + "| **Outcomes (MACE)** | AMI, stroke, CHF — from `v_inpatient_services` after exposure date |\n", + "| **Follow-up** | Months of continuous enrollment after second-line start |" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b1c2d303", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from ehrdata.io.source.adapters import lced\n", + "from ehrdata.io.source.vocab import ndc as ndc_vocab, rxnorm, loinc as loinc_vocab" + ] + }, + { + "cell_type": "markdown", + "id": "c1d2e303", + "metadata": {}, + "source": [ + "## 1. Synthetic LCED Tables\n", + "\n", + "Below we build synthetic DataFrames that mirror the LCED schema. Replace these with real `v_*` table pulls in a live environment." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d1e2f303", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Enrollment rows : 20\n", + "Outpatient dx rows : 62\n", + "Drug claim rows : 40\n", + "Observation rows : 60\n", + "Inpatient rows : 10\n" + ] + } + ], + "source": [ + "np.random.seed(0)\n", + "\n", + "N_PAT = 20\n", + "pids = [f\"P{i:04d}\" for i in range(1, N_PAT + 1)]\n", + "\n", + "# --- Enrollment summary ---\n", + "v_annual = pd.DataFrame({\n", + " \"patient_id\": pids,\n", + " \"dobyr\": np.random.randint(1940, 1975, N_PAT),\n", + " \"sex\": np.random.choice([\"M\", \"F\"], N_PAT),\n", + " **{f\"enrind{m}\": np.random.choice([0, 1], N_PAT, p=[0.05, 0.95]) for m in range(1, 13)},\n", + " \"year\": [2015] * N_PAT,\n", + "})\n", + "\n", + "# --- Outpatient services (diagnosis) ---\n", + "_dx_pool = [\n", + " (\"E119\", \"0\"), (\"E118\", \"0\"), (\"25000\", \"9\"), (\"25002\", \"9\"),\n", + " (\"I509\", \"0\"), (\"41001\", \"9\"), (\"I259\", \"0\"), (\"Z794\", \"0\"),\n", + "]\n", + "\n", + "rows = []\n", + "for pid in pids:\n", + " for _ in range(np.random.randint(2, 6)):\n", + " dx, dxver = _dx_pool[np.random.randint(len(_dx_pool))]\n", + " rows.append({\"patient_id\": pid,\n", + " \"svcdate\": pd.Timestamp(\"2013-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1000))),\n", + " \"dx1\": dx, \"dxver\": dxver})\n", + "v_outpatient = pd.DataFrame(rows)\n", + "\n", + "# --- Drug claims ---\n", + "# NDC codes: metformin, GLP-1s, SGLT-2s, DPP-4s, SUs (synthetic 11-digit codes)\n", + "MET_NDCS = [\"00093717401\", \"00093717501\"]\n", + "GLP1_NDCS = [\"00169368812\", \"00169368915\"] # exenatide, liraglutide\n", + "SGLT2_NDCS = [\"00310653090\", \"57844011290\"] # dapagliflozin, canagliflozin\n", + "DPP4_NDCS = [\"00006054054\", \"00310027190\"] # sitagliptin, saxagliptin\n", + "SU_NDCS = [\"00093116401\", \"00093013901\"] # glimepiride, glyburide\n", + "\n", + "SECOND_LINE = GLP1_NDCS + SGLT2_NDCS + DPP4_NDCS + SU_NDCS\n", + "DRUG_CLASS = (\n", + " {n: \"glp1\" for n in GLP1_NDCS} |\n", + " {n: \"sglt2\" for n in SGLT2_NDCS} |\n", + " {n: \"dpp4\" for n in DPP4_NDCS} |\n", + " {n: \"su\" for n in SU_NDCS}\n", + ")\n", + "\n", + "drg_rows = []\n", + "for pid in np.random.choice(pids, 40, replace=True):\n", + " pool = MET_NDCS + SECOND_LINE\n", + " ndc = pool[np.random.randint(len(pool))]\n", + " date = pd.Timestamp(\"2014-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1200)))\n", + " drg_rows.append({\"patient_id\": pid, \"svcdate\": date,\n", + " \"ndcnum\": ndc, \"daysupp\": 30, \"refill\": 0})\n", + "v_drug_claims = pd.DataFrame(drg_rows)\n", + "\n", + "# --- Observations (HbA1c) ---\n", + "HBA1C_LOINCS = [\"4548-4\", \"17856-6\"]\n", + "obs_rows = []\n", + "for pid in np.random.choice(pids, 60, replace=True):\n", + " loinc = np.random.choice(HBA1C_LOINCS)\n", + " value = round(np.random.uniform(5.5, 11.0), 1)\n", + " date = pd.Timestamp(\"2014-06-01\") + pd.Timedelta(days=int(np.random.randint(0, 900)))\n", + " obs_rows.append({\"patient_id\": pid, \"observation_date\": date,\n", + " \"loinc_test_id\": loinc, \"std_value\": value, \"std_uom\": \"%\"})\n", + "v_observation = pd.DataFrame(obs_rows)\n", + "\n", + "# --- Inpatient services (MACE outcomes) ---\n", + "MACE_CODES = [\"I219\", \"I639\", \"I509\", \"41001\", \"43401\"]\n", + "inp_rows = []\n", + "for pid in np.random.choice(pids, 10, replace=True):\n", + " dx = MACE_CODES[np.random.randint(len(MACE_CODES))]\n", + " date = pd.Timestamp(\"2015-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1000)))\n", + " inp_rows.append({\"patient_id\": pid, \"svcdate\": date, \"dx1\": dx, \"dx2\": None})\n", + "v_inpatient_services = pd.DataFrame(inp_rows)\n", + "\n", + "print(f\"Enrollment rows : {len(v_annual)}\")\n", + "print(f\"Outpatient dx rows : {len(v_outpatient)}\")\n", + "print(f\"Drug claim rows : {len(v_drug_claims)}\")\n", + "print(f\"Observation rows : {len(v_observation)}\")\n", + "print(f\"Inpatient rows : {len(v_inpatient_services)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a303", + "metadata": {}, + "source": [ + "## 2. Eligible Population\n", + "\n", + "### Step 2a — T2D Cohort\n", + "\n", + "Patients with ≥ 1 T2D code across any outpatient, inpatient, or facility table." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f1a2b303", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Patients with ≥1 T2D code: 17\n" + ] + } + ], + "source": [ + "T2D_ICD9 = [\"25000\", \"25002\", \"25010\", \"25012\", \"25040\", \"25042\",\n", + " \"25050\", \"25052\", \"25060\", \"25062\", \"25070\", \"25072\",\n", + " \"25090\", \"25092\"]\n", + "\n", + "def is_t2d(dx_series):\n", + " nodot = dx_series.str.replace(\".\", \"\", regex=False)\n", + " icd9 = nodot.isin(T2D_ICD9)\n", + " icd10 = dx_series.str.startswith(\"E11\", na=False)\n", + " return icd9 | icd10\n", + "\n", + "t2d_flag = is_t2d(v_outpatient[\"dx1\"])\n", + "t2d_patients = v_outpatient[t2d_flag][\"patient_id\"].unique()\n", + "print(f\"Patients with ≥1 T2D code: {len(t2d_patients)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c403", + "metadata": {}, + "source": [ + "### Step 2b — Age Filter (≥ 45 years at index)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b2c3d403", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "T2D patients aged ≥ 45: 17\n" + ] + } + ], + "source": [ + "# Compute age at 2014-01-01 (the study start year in the original R analysis)\n", + "STUDY_START = pd.Timestamp(\"2014-01-01\")\n", + "enroll = v_annual[[\"patient_id\", \"dobyr\"]].drop_duplicates()\n", + "enroll[\"age_at_study\"] = STUDY_START.year - enroll[\"dobyr\"]\n", + "\n", + "old_patients = enroll[enroll[\"age_at_study\"] >= 45][\"patient_id\"].values\n", + "eligible = np.intersect1d(t2d_patients, old_patients)\n", + "print(f\"T2D patients aged ≥ 45: {len(eligible)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e403", + "metadata": {}, + "source": [ + "## 3. Metformin Index Date\n", + "\n", + "The index date is the **first** metformin claim date per patient." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d2e3f403", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Patients with metformin: 4\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_idmetformin_start
0P00012016-01-01
1P00052016-11-25
2P00072015-03-25
3P00202016-04-28
\n", + "
" + ], + "text/plain": [ + " patient_id metformin_start\n", + "0 P0001 2016-01-01\n", + "1 P0005 2016-11-25\n", + "2 P0007 2015-03-25\n", + "3 P0020 2016-04-28" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "met_claims = v_drug_claims[\n", + " v_drug_claims[\"patient_id\"].isin(eligible) &\n", + " v_drug_claims[\"ndcnum\"].isin(MET_NDCS)\n", + "].copy()\n", + "\n", + "met_index = (\n", + " met_claims\n", + " .groupby(\"patient_id\")[\"svcdate\"]\n", + " .min()\n", + " .rename(\"metformin_start\")\n", + " .reset_index()\n", + ")\n", + "\n", + "print(f\"Patients with metformin: {len(met_index)}\")\n", + "met_index.head()" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a403", + "metadata": {}, + "source": [ + "## 4. Poor Glycaemic Control\n", + "\n", + "Filter HbA1c observations to:\n", + "1. After metformin start\n", + "2. After 2014-01-01 (study window)\n", + "3. HbA1c ≥ 7 %\n", + "\n", + "Then flag poor control: value ≥ 9 % **OR** two consecutive measurements ≤ 183 days apart.\n", + "Keep only the **first** qualifying observation per patient." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f2a3b403", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Patients with poor glycaemic control: 0\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_idpoor_ctrl_datehba1cmetformin_start
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [patient_id, poor_ctrl_date, hba1c, metformin_start]\n", + "Index: []" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a1c = (\n", + " v_observation[v_observation[\"loinc_test_id\"].isin(HBA1C_LOINCS)]\n", + " .copy()\n", + " .rename(columns={\"observation_date\": \"obs_date\", \"std_value\": \"hba1c\"})\n", + " [[\"patient_id\", \"obs_date\", \"hba1c\"]]\n", + ")\n", + "\n", + "# Merge with metformin index\n", + "a1c = a1c.merge(met_index, on=\"patient_id\", how=\"inner\")\n", + "\n", + "# Apply filters: after metformin start, after study start, HbA1c ≥ 7\n", + "a1c = a1c[\n", + " (a1c[\"obs_date\"] > a1c[\"metformin_start\"]) &\n", + " (a1c[\"obs_date\"] > STUDY_START) &\n", + " (a1c[\"hba1c\"] >= 7.0)\n", + "].sort_values([\"patient_id\", \"obs_date\"]).reset_index(drop=True)\n", + "\n", + "# Days between consecutive measurements per patient\n", + "a1c[\"prev_date\"] = a1c.groupby(\"patient_id\")[\"obs_date\"].shift(1)\n", + "a1c[\"days_since_last\"] = (a1c[\"obs_date\"] - a1c[\"prev_date\"]).dt.days\n", + "\n", + "# Poor control flag\n", + "a1c[\"poor_control\"] = (a1c[\"hba1c\"] >= 9.0) | (a1c[\"days_since_last\"] <= 183)\n", + "\n", + "# First qualifying poor-control event per patient\n", + "poor_ctrl = (\n", + " a1c[a1c[\"poor_control\"]]\n", + " .groupby(\"patient_id\")\n", + " .first()\n", + " .reset_index()\n", + " [[\"patient_id\", \"obs_date\", \"hba1c\", \"metformin_start\"]]\n", + " .rename(columns={\"obs_date\": \"poor_ctrl_date\"})\n", + ")\n", + "\n", + "print(f\"Patients with poor glycaemic control: {len(poor_ctrl)}\")\n", + "poor_ctrl.head()" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c503", + "metadata": {}, + "source": [ + "## 5. Second-Line Therapy Assignment\n", + "\n", + "For each patient in the poor-control cohort, find the **first** second-line drug claim that:\n", + "- Falls **after** the poor-control HbA1c date\n", + "- Falls **within 365 days** of the poor-control date\n", + "- Has a gap of > 180 days from any previous second-line claim (new initiation, not refill)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b3c4d503", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Final cohort size: 0\n", + "\n", + "Drug class distribution:\n", + "Series([], )\n" + ] + } + ], + "source": [ + "sl_claims = v_drug_claims[\n", + " v_drug_claims[\"patient_id\"].isin(poor_ctrl[\"patient_id\"]) &\n", + " v_drug_claims[\"ndcnum\"].isin(SECOND_LINE)\n", + "].copy()\n", + "sl_claims[\"drug_class\"] = sl_claims[\"ndcnum\"].map(DRUG_CLASS)\n", + "\n", + "# Merge with poor-control date\n", + "sl_claims = sl_claims.merge(\n", + " poor_ctrl[[\"patient_id\", \"poor_ctrl_date\"]], on=\"patient_id\", how=\"inner\"\n", + ")\n", + "\n", + "# Filter: after poor-control date and within 365 days\n", + "sl_claims[\"days_after_ctrl\"] = (sl_claims[\"svcdate\"] - sl_claims[\"poor_ctrl_date\"]).dt.days\n", + "sl_claims = sl_claims[\n", + " (sl_claims[\"days_after_ctrl\"] >= 0) &\n", + " (sl_claims[\"days_after_ctrl\"] <= 365)\n", + "].sort_values([\"patient_id\", \"svcdate\"]).reset_index(drop=True)\n", + "\n", + "# Keep first claim per patient (new initiation)\n", + "sl_claims[\"prev_sl_date\"] = sl_claims.groupby(\"patient_id\")[\"svcdate\"].shift(1)\n", + "sl_claims[\"sl_gap\"] = (sl_claims[\"svcdate\"] - sl_claims[\"prev_sl_date\"]).dt.days\n", + "sl_claims[\"is_new\"] = sl_claims[\"prev_sl_date\"].isna() | (sl_claims[\"sl_gap\"] > 180)\n", + "\n", + "cohort = (\n", + " sl_claims[sl_claims[\"is_new\"]]\n", + " .groupby(\"patient_id\")\n", + " .first()\n", + " .reset_index()\n", + " [[\"patient_id\", \"svcdate\", \"drug_class\", \"poor_ctrl_date\"]]\n", + " .rename(columns={\"svcdate\": \"second_line_start\"})\n", + ")\n", + "\n", + "print(f\"Final cohort size: {len(cohort)}\")\n", + "print()\n", + "print(\"Drug class distribution:\")\n", + "print(cohort[\"drug_class\"].value_counts().to_string())" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e503", + "metadata": {}, + "source": [ + "## 6. MACE Outcome Ascertainment\n", + "\n", + "MACE (Major Adverse Cardiovascular Events) = AMI, Stroke, or CHF from inpatient services after second-line start." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d3e4f503", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AMI : 0 patients\n", + "STROKE: 0 patients\n", + "CHF : 0 patients\n" + ] + } + ], + "source": [ + "AMI_ICD9_PFX = [\"410\"]\n", + "AMI_ICD10_PFX = [\"I21\", \"I22\"]\n", + "STROKE_ICD9_PFX = [\"160\", \"161\", \"162\", \"163\", \"430\", \"431\", \"432\", \"433\", \"434\", \"435\"]\n", + "STROKE_ICD10_PFX= [\"I60\", \"I61\", \"I62\", \"I63\", \"I64\"]\n", + "CHF_ICD9 = [\"4280\", \"4281\", \"4282\", \"4283\", \"4284\", \"4289\"]\n", + "CHF_ICD10_PFX = [\"I50\"]\n", + "\n", + "def flag_outcome(dx_series, icd9_pfx=None, icd10_pfx=None, icd9_exact=None):\n", + " m = pd.Series(False, index=dx_series.index)\n", + " if icd9_pfx:\n", + " for p in icd9_pfx:\n", + " m |= dx_series.str.startswith(p, na=False)\n", + " if icd10_pfx:\n", + " for p in icd10_pfx:\n", + " m |= dx_series.str.startswith(p, na=False)\n", + " if icd9_exact:\n", + " m |= dx_series.isin(icd9_exact)\n", + " return m\n", + "\n", + "inp = v_inpatient_services.copy()\n", + "\n", + "for col in [\"dx1\", \"dx2\"]:\n", + " if col not in inp.columns:\n", + " inp[col] = pd.NA\n", + "\n", + "inp[\"ami\"] = flag_outcome(inp[\"dx1\"], icd9_pfx=AMI_ICD9_PFX, icd10_pfx=AMI_ICD10_PFX)\n", + "inp[\"stroke\"] = flag_outcome(inp[\"dx1\"], icd9_pfx=STROKE_ICD9_PFX, icd10_pfx=STROKE_ICD10_PFX)\n", + "inp[\"chf\"] = flag_outcome(inp[\"dx1\"], icd9_exact=CHF_ICD9, icd10_pfx=CHF_ICD10_PFX)\n", + "inp[\"mace\"] = inp[\"ami\"] | inp[\"stroke\"] | inp[\"chf\"]\n", + "\n", + "# Merge with cohort and keep events after second-line start\n", + "inp_cohort = (\n", + " inp[inp[\"mace\"] & inp[\"patient_id\"].isin(cohort[\"patient_id\"])]\n", + " .merge(cohort[[\"patient_id\", \"second_line_start\"]], on=\"patient_id\", how=\"inner\")\n", + ")\n", + "inp_cohort = inp_cohort[inp_cohort[\"svcdate\"] > inp_cohort[\"second_line_start\"]]\n", + "\n", + "for outcome in [\"ami\", \"stroke\", \"chf\"]:\n", + " n = inp_cohort[inp_cohort[outcome]][\"patient_id\"].nunique()\n", + " print(f\"{outcome.upper():6s}: {n} patients\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a503", + "metadata": {}, + "source": [ + "## 7. Coverage / Follow-Up Calculation\n", + "\n", + "Follow-up is defined as the number of continuous enrolled months after the second-line start date. Enrollment gaps are encoded as `enrind{1..12} == 0` in `v_annual_summary_enrollment`." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "f3a4b503", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Follow-up summary (months after second-line start):\n", + "count 0.0\n", + "mean NaN\n", + "std NaN\n", + "min NaN\n", + "25% NaN\n", + "50% NaN\n", + "75% NaN\n", + "max NaN\n", + "Name: followup_months, dtype: float64\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_iddrug_classsecond_line_startfollowup_months
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [patient_id, drug_class, second_line_start, followup_months]\n", + "Index: []" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "enrind_cols = [f\"enrind{m}\" for m in range(1, 13)]\n", + "\n", + "def followup_months(pid, start_date, enrollment_df, enrind_cols):\n", + " \"\"\"Return months of continuous enrollment after start_date.\"\"\"\n", + " rows = enrollment_df[enrollment_df[\"patient_id\"] == pid].sort_values(\"year\")\n", + " count = 0\n", + " for _, row in rows.iterrows():\n", + " for m_idx, col in enumerate(enrind_cols, start=1):\n", + " month_date = pd.Timestamp(year=int(row[\"year\"]), month=m_idx, day=1)\n", + " if month_date < start_date:\n", + " continue\n", + " if row[col] == 0:\n", + " return count\n", + " count += 1\n", + " return count\n", + "\n", + "cohort_follow = cohort.copy()\n", + "cohort_follow[\"followup_months\"] = cohort_follow.apply(\n", + " lambda r: followup_months(r[\"patient_id\"], r[\"second_line_start\"], v_annual, enrind_cols),\n", + " axis=1,\n", + ")\n", + "\n", + "print(\"Follow-up summary (months after second-line start):\")\n", + "print(cohort_follow[\"followup_months\"].describe().round(1))\n", + "cohort_follow[[\"patient_id\", \"drug_class\", \"second_line_start\", \"followup_months\"]].head(8)" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c603", + "metadata": {}, + "source": [ + "## 8. EHRData Bridge\n", + "\n", + "Convert the final cohort to an `EHRData` object for downstream ML / survival analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b4c5d603", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EHRData object with n_obs × n_vars = 0 × 0\n", + " obs: 'dobyr', 'sex'\n", + " var: 'concept_source', 'concept_code'\n", + " uns: 'source_io_source', 'source_io_tables'\n", + " shape of .X: (0, 0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/functools.py:909: ImplicitModificationWarning: Transforming to str index.\n", + " return dispatch(args[0].__class__)(*args, **kw)\n" + ] + } + ], + "source": [ + "from ehrdata.io.source import to_ehrdata\n", + "\n", + "# Build canonical tables for cohort patients only\n", + "cohort_ids = set(cohort[\"patient_id\"])\n", + "\n", + "# patinfo from enrollment\n", + "patinfo = (\n", + " v_annual[[\"patient_id\", \"dobyr\", \"sex\"]]\n", + " .drop_duplicates(subset=\"patient_id\")\n", + " .query(\"patient_id in @cohort_ids\")\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "# diagnosis canonical table\n", + "diag_raw = v_outpatient[[\"patient_id\", \"svcdate\", \"dx1\", \"dxver\"]].copy()\n", + "diag_raw = diag_raw[diag_raw[\"patient_id\"].isin(cohort_ids)]\n", + "diag_raw = diag_raw.rename(columns={\"svcdate\": \"eventdate\", \"dx1\": \"dx\"})\n", + "diag_raw[\"dxver\"] = diag_raw[\"dxver\"].fillna(\n", + " diag_raw[\"dx\"].apply(lambda d: \"9\" if d and d[0].isdigit() else \"0\")\n", + ")\n", + "\n", + "# therapy canonical table from drug claims\n", + "therapy_raw = (\n", + " v_drug_claims[v_drug_claims[\"patient_id\"].isin(cohort_ids)]\n", + " .rename(columns={\"svcdate\": \"fill_date\", \"ndcnum\": \"ndc11\"})\n", + " .assign(\n", + " ingredient=lambda df: df[\"ndc11\"].map(DRUG_CLASS).fillna(\"metformin\"),\n", + " rxcui=None,\n", + " prescription_date=pd.NaT,\n", + " start_date=pd.NaT,\n", + " end_date=pd.NaT,\n", + " refill=pd.array([pd.NA] * len(v_drug_claims[v_drug_claims[\"patient_id\"].isin(cohort_ids)]), dtype=\"Int64\"),\n", + " )\n", + " [[\"patient_id\", \"fill_date\", \"ingredient\", \"ndc11\", \"rxcui\",\n", + " \"prescription_date\", \"start_date\", \"end_date\", \"refill\"]]\n", + ")\n", + "\n", + "edata = to_ehrdata(\n", + " patinfo,\n", + " diagnosis=diag_raw,\n", + " therapy=therapy_raw,\n", + " source=\"lced\",\n", + ")\n", + "print(edata)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "c4d5e603", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "obs columns: ['dobyr', 'sex', 'drug_class', 'second_line_start', 'followup_months']\n", + "\n", + "Empty DataFrame\n", + "Columns: [dobyr, sex, drug_class, second_line_start, followup_months]\n", + "Index: []\n" + ] + } + ], + "source": [ + "# Attach cohort metadata to obs\n", + "cohort_meta = (\n", + " cohort_follow[[\"patient_id\", \"drug_class\", \"second_line_start\", \"followup_months\"]]\n", + " .set_index(\"patient_id\")\n", + ")\n", + "# Align to edata.obs index\n", + "common = edata.obs_names.intersection(cohort_meta.index)\n", + "edata.obs.loc[common, \"drug_class\"] = cohort_meta.loc[common, \"drug_class\"]\n", + "edata.obs.loc[common, \"second_line_start\"] = cohort_meta.loc[common, \"second_line_start\"]\n", + "edata.obs.loc[common, \"followup_months\"] = cohort_meta.loc[common, \"followup_months\"]\n", + "\n", + "print(\"obs columns:\", edata.obs.columns.tolist())\n", + "print()\n", + "print(edata.obs.head(5).to_string())" + ] + }, + { + "cell_type": "markdown", + "id": "d4e5f603", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Cohort step | Key operation |\n", + "|---|---|\n", + "| T2D definition | `dx.str.startswith(\"E11\")` \\| `dx.isin(T2D_ICD9)` |\n", + "| Age filter | enrollment `dobyr` vs study year |\n", + "| Metformin index date | `groupby(\"patient_id\")[\"svcdate\"].min()` |\n", + "| Poor control HbA1c | `groupby` + `shift` + `diff` on sorted dates |\n", + "| Second-line assignment | NDC lookup + 365-day window + 180-day gap filter |\n", + "| MACE outcomes | `str.startswith` on ICD prefix lists |\n", + "| Follow-up | `enrind{1..12}` monthly enrollment scan |\n", + "| EHRData export | `to_ehrdata(patinfo, diagnosis=..., therapy=...)` |\n", + "\n", + "**Next steps**: Attach MACE outcomes as time-to-event columns in `edata.obs` and fit a survival model using `lifelines` or `scikit-survival`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/source_loinc_mapping.ipynb b/docs/tutorials/source_loinc_mapping.ipynb new file mode 100644 index 00000000..032704e9 --- /dev/null +++ b/docs/tutorials/source_loinc_mapping.ipynb @@ -0,0 +1,806 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a1b2c301", + "metadata": {}, + "source": [ + "# LOINC Mapping in LCED Lab Data\n", + "\n", + "**LOINC** (Logical Observation Identifiers Names and Codes) is the standard coding system for laboratory and clinical observations. In IBM LCED, two sources carry LOINC codes:\n", + "\n", + "| Source | Column | Domain |\n", + "|---|---|---|\n", + "| `v_observation` (Explorys EMR) | `loinc_test_id` | Structured EMR lab values |\n", + "| `v_lab_results` (MarketScan claims) | `loinccd` | Claims-based lab results |\n", + "\n", + "This notebook shows how to search for clinical concepts by LOINC, filter lab results, and join across sources — translating the original R `labtest code.R` workflow." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b1c2d301", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from ehrdata.io.source.adapters import lced\n", + "from ehrdata.io.source.vocab import loinc as loinc_vocab" + ] + }, + { + "cell_type": "markdown", + "id": "c1d2e301", + "metadata": {}, + "source": [ + "## 1. LOINC Reference Data\n", + "\n", + "The `loinc.load_loinc_map` function reads a LOINC reference table and returns a DataFrame with at minimum `loinc_code` and `long_common_name` columns.\n", + "\n", + "The original R code queried this from the database via:\n", + "```r\n", + "loinc.df <- dbGetQuery(conn, \"SELECT * FROM xref.loinc\")\n", + "```\n", + "\n", + "In Python we provide it as a CSV file. Below we build a representative subset covering the most common clinical concepts in diabetes research." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d1e2f301", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reference LOINC codes: 20\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
loinc_codelong_common_name
04548-4Hemoglobin A1c/Hemoglobin.total in Blood
117856-6Hemoglobin A1c/Hemoglobin.total in Blood by HPLC
259261-8Hemoglobin A1c/Hemoglobin.total in Blood by Im...
329463-7Body weight
43141-9Body weight Measured
575292-3Body weight - Reported
68302-2Body height
73137-7Body height Measured
\n", + "
" + ], + "text/plain": [ + " loinc_code long_common_name\n", + "0 4548-4 Hemoglobin A1c/Hemoglobin.total in Blood\n", + "1 17856-6 Hemoglobin A1c/Hemoglobin.total in Blood by HPLC\n", + "2 59261-8 Hemoglobin A1c/Hemoglobin.total in Blood by Im...\n", + "3 29463-7 Body weight\n", + "4 3141-9 Body weight Measured\n", + "5 75292-3 Body weight - Reported\n", + "6 8302-2 Body height\n", + "7 3137-7 Body height Measured" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Representative LOINC codes used in T2D / cardiometabolic research\n", + "# In practice, load from loinc_map.csv (subset of the full LOINC table)\n", + "loinc_reference = pd.DataFrame({\n", + " \"loinc_code\": [\n", + " \"4548-4\", \"17856-6\", \"59261-8\", # HbA1c\n", + " \"29463-7\", \"3141-9\", \"75292-3\", # Weight\n", + " \"8302-2\", \"3137-7\", \"8308-9\", # Height\n", + " \"39156-5\", \"59574-4\", \"89270-3\", # BMI\n", + " \"2345-7\", \"14749-6\", # Glucose\n", + " \"2160-0\", \"38483-4\", # Creatinine\n", + " \"4544-3\", \"20570-8\", # Haematocrit\n", + " \"718-7\", \"26515-7\", # Haemoglobin, Platelets\n", + " ],\n", + " \"long_common_name\": [\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood\",\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood by HPLC\",\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood by Immunoassay\",\n", + " \"Body weight\",\n", + " \"Body weight Measured\",\n", + " \"Body weight - Reported\",\n", + " \"Body height\",\n", + " \"Body height Measured\",\n", + " \"Body height - standing\",\n", + " \"Body mass index (BMI) [Ratio]\",\n", + " \"Body mass index (BMI) [Percentile]\",\n", + " \"Body mass index (BMI) [Ratio] Reported\",\n", + " \"Glucose [Mass/volume] in Serum or Plasma\",\n", + " \"Glucose [Moles/volume] in Serum or Plasma\",\n", + " \"Creatinine [Mass/volume] in Serum or Plasma\",\n", + " \"Creatinine [Mass/volume] in Blood\",\n", + " \"Hematocrit [Volume Fraction] of Blood by Automated count\",\n", + " \"Hematocrit [Volume Fraction] of Blood\",\n", + " \"Hemoglobin [Mass/volume] in Blood\",\n", + " \"Platelets [#/volume] in Blood by Automated count\",\n", + " ],\n", + "})\n", + "\n", + "print(f\"Reference LOINC codes: {len(loinc_reference)}\")\n", + "loinc_reference.head(8)" + ] + }, + { + "cell_type": "markdown", + "id": "e1f2a301", + "metadata": {}, + "source": [ + "## 2. Concept Search\n", + "\n", + "The R pipeline used `grep` to search concept names. Python's `str.contains` is the direct equivalent." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f1a2b301", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== HbA1c ===\n", + " loinc_code long_common_name\n", + "0 4548-4 Hemoglobin A1c/Hemoglobin.total in Blood\n", + "1 17856-6 Hemoglobin A1c/Hemoglobin.total in Blood by HPLC\n", + "2 59261-8 Hemoglobin A1c/Hemoglobin.total in Blood by Im...\n", + "\n", + "=== Body Weight ===\n", + " loinc_code long_common_name\n", + "0 29463-7 Body weight\n", + "1 3141-9 Body weight Measured\n", + "2 75292-3 Body weight - Reported\n", + "\n", + "=== Body Height ===\n", + " loinc_code long_common_name\n", + "0 8302-2 Body height\n", + "1 3137-7 Body height Measured\n", + "2 8308-9 Body height - standing\n", + "\n", + "=== BMI ===\n", + " loinc_code long_common_name\n", + "0 39156-5 Body mass index (BMI) [Ratio]\n", + "1 59574-4 Body mass index (BMI) [Percentile]\n", + "2 89270-3 Body mass index (BMI) [Ratio] Reported\n" + ] + } + ], + "source": [ + "def search_loinc(df, pattern, col=\"long_common_name\"):\n", + " \"\"\"Case-insensitive substring search over LOINC names.\"\"\"\n", + " mask = df[col].str.contains(pattern, case=False, na=False)\n", + " return df[mask].reset_index(drop=True)\n", + "\n", + "\n", + "# --- HbA1c ---\n", + "print(\"=== HbA1c ===\")\n", + "print(search_loinc(loinc_reference, r\"hba1c|hemoglobin a1c|haemoglobin a1c\")[[\"loinc_code\", \"long_common_name\"]])\n", + "print()\n", + "\n", + "# --- Weight ---\n", + "print(\"=== Body Weight ===\")\n", + "print(search_loinc(loinc_reference, r\"body weight\")[[\"loinc_code\", \"long_common_name\"]])\n", + "print()\n", + "\n", + "# --- Height ---\n", + "print(\"=== Body Height ===\")\n", + "print(search_loinc(loinc_reference, r\"body height\")[[\"loinc_code\", \"long_common_name\"]])\n", + "print()\n", + "\n", + "# --- BMI ---\n", + "print(\"=== BMI ===\")\n", + "print(search_loinc(loinc_reference, r\"body mass index|\\bbmi\\b\")[[\"loinc_code\", \"long_common_name\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "a2b3c401", + "metadata": {}, + "source": [ + "## 3. Key LOINC Codes for Diabetes Research\n", + "\n", + "Based on the original R analysis, the following LOINC codes are used most frequently:\n", + "\n", + "| Concept | Primary LOINC | Common variants |\n", + "|---|---|---|\n", + "| HbA1c | **4548-4** | 17856-6, 59261-8 |\n", + "| Body weight | **29463-7** | 3141-9, 75292-3 |\n", + "| Body height | **8302-2** | 3137-7, 8308-9 |\n", + "| BMI | **39156-5** | 59574-4, 89270-3 |\n", + "\n", + "> **Tip**: When filtering `v_observation`, use all known variants of a concept to maximise recall. `4548-4` alone captures ~80 % of HbA1c results in most EMR datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b2c3d401", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hba1c : ['4548-4', '17856-6', '59261-8']\n", + "weight : ['29463-7', '3141-9', '75292-3']\n", + "height : ['8302-2', '3137-7', '8308-9']\n", + "bmi : ['39156-5', '59574-4', '89270-3']\n" + ] + } + ], + "source": [ + "# Canonical concept code lists (union of variants)\n", + "HBA1C_LOINCS = [\"4548-4\", \"17856-6\", \"59261-8\"]\n", + "WEIGHT_LOINCS = [\"29463-7\", \"3141-9\", \"75292-3\"]\n", + "HEIGHT_LOINCS = [\"8302-2\", \"3137-7\", \"8308-9\"]\n", + "BMI_LOINCS = [\"39156-5\", \"59574-4\", \"89270-3\"]\n", + "\n", + "all_concept_loincs = {\n", + " \"hba1c\": HBA1C_LOINCS,\n", + " \"weight\": WEIGHT_LOINCS,\n", + " \"height\": HEIGHT_LOINCS,\n", + " \"bmi\": BMI_LOINCS,\n", + "}\n", + "for concept, codes in all_concept_loincs.items():\n", + " print(f\"{concept:8s}: {codes}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2d3e401", + "metadata": {}, + "source": [ + "## 4. Synthetic LCED Lab Data\n", + "\n", + "Below we simulate the two lab sources in LCED so you can follow the filtering steps without raw data access." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d2e3f401", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "v_observation rows : 30\n", + "v_lab_results rows : 20\n" + ] + } + ], + "source": [ + "np.random.seed(42)\n", + "n = 30\n", + "\n", + "patient_pool = [f\"P{i:04d}\" for i in range(1, 11)]\n", + "\n", + "# Explorys v_observation (EMR)\n", + "v_observation = pd.DataFrame({\n", + " \"patient_id\": np.random.choice(patient_pool, n),\n", + " \"observation_date\": pd.date_range(\"2014-01-01\", periods=n, freq=\"14D\"),\n", + " \"loinc_test_id\": np.random.choice(\n", + " HBA1C_LOINCS + WEIGHT_LOINCS + HEIGHT_LOINCS + BMI_LOINCS + [\"2345-7\", \"2160-0\"],\n", + " n,\n", + " ),\n", + " \"std_value\": np.round(np.random.uniform(4.0, 12.0, n), 1),\n", + " \"std_uom\": np.random.choice([\"%\", \"kg\", \"cm\", \"kg/m2\", \"mg/dL\"], n),\n", + "})\n", + "\n", + "# MarketScan v_lab_results (claims)\n", + "n2 = 20\n", + "v_lab_results = pd.DataFrame({\n", + " \"patient_id\": np.random.choice(patient_pool, n2),\n", + " \"svcdate\": pd.date_range(\"2014-03-01\", periods=n2, freq=\"21D\"),\n", + " \"loinccd\": np.random.choice(HBA1C_LOINCS + WEIGHT_LOINCS + [\"2345-7\"], n2),\n", + " \"result\": np.round(np.random.uniform(4.0, 12.0, n2), 1),\n", + " \"resunit\": np.random.choice([\"%\", \"kg\", \"mg/dL\"], n2),\n", + " \"resltcat\": np.random.choice([None, \"Normal\", \"High\", \"Low\"], n2),\n", + "})\n", + "\n", + "print(f\"v_observation rows : {len(v_observation)}\")\n", + "print(f\"v_lab_results rows : {len(v_lab_results)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e2f3a401", + "metadata": {}, + "source": [ + "## 5. Building the Canonical Lab Table\n", + "\n", + "`lced.build_labtest` unifies both sources into a single canonical schema:\n", + "\n", + "| Column | Source | Notes |\n", + "|---|---|---|\n", + "| `patient_id` | both | |\n", + "| `eventdate` | both | `observation_date` or `svcdate` |\n", + "| `value` | both | `std_value` or `result` |\n", + "| `unit` | both | `std_uom` or `resunit` |\n", + "| `valuecat` | v_lab_results | `resltcat` (only 3 % non-missing) |\n", + "| `loinc` | both | `loinc_test_id` or `loinccd` |" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f2a3b401", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Combined lab rows: 50\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_ideventdatevaluevaluecatunitloinc
0P00012014-04-126.0Normal%2345-7
1P00012014-10-226.2Nonecm89270-3
2P00012014-12-177.4Nonecm3137-7
3P00022014-08-1310.0Nonecm39156-5
4P00022014-09-2411.7Nonecm29463-7
5P00022014-11-085.2High%4548-4
\n", + "
" + ], + "text/plain": [ + " patient_id eventdate value valuecat unit loinc\n", + "0 P0001 2014-04-12 6.0 Normal % 2345-7\n", + "1 P0001 2014-10-22 6.2 None cm 89270-3\n", + "2 P0001 2014-12-17 7.4 None cm 3137-7\n", + "3 P0002 2014-08-13 10.0 None cm 39156-5\n", + "4 P0002 2014-09-24 11.7 None cm 29463-7\n", + "5 P0002 2014-11-08 5.2 High % 4548-4" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labs = lced.build_labtest(v_observation, v_lab_results)\n", + "print(f\"Combined lab rows: {len(labs)}\")\n", + "labs.head(6)" + ] + }, + { + "cell_type": "markdown", + "id": "a3b4c501", + "metadata": {}, + "source": [ + "## 6. Filtering by LOINC Concept\n", + "\n", + "With the canonical `loinc` column present, filtering to a specific concept is a single boolean mask — equivalent to the R `WHERE a.loinc_test_id IN (...)` query." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b3c4d501", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HbA1c observations: 17\n", + "Patients with HbA1c: 8\n", + "\n", + " patient_id eventdate value unit loinc\n", + "5 P0002 2014-11-08 5.2 % 4548-4\n", + "8 P0003 2014-07-02 8.2 mg/dL 17856-6\n", + "11 P0004 2014-01-15 10.0 cm 59261-8\n", + "14 P0004 2014-05-24 5.8 kg 4548-4\n", + "17 P0004 2015-02-21 4.3 mg/dL 4548-4\n", + "18 P0005 2014-02-12 5.7 cm 59261-8\n", + "20 P0005 2014-06-14 8.5 mg/dL 4548-4\n", + "25 P0006 2014-09-10 8.7 % 17856-6\n" + ] + } + ], + "source": [ + "# HbA1c results\n", + "hba1c_labs = labs[labs[\"loinc\"].isin(HBA1C_LOINCS)].copy()\n", + "hba1c_labs[\"value\"] = pd.to_numeric(hba1c_labs[\"value\"], errors=\"coerce\")\n", + "\n", + "print(f\"HbA1c observations: {len(hba1c_labs)}\")\n", + "print(f\"Patients with HbA1c: {hba1c_labs['patient_id'].nunique()}\")\n", + "print()\n", + "print(hba1c_labs[[\"patient_id\", \"eventdate\", \"value\", \"unit\", \"loinc\"]].head(8))" + ] + }, + { + "cell_type": "markdown", + "id": "c3d4e501", + "metadata": {}, + "source": [ + "## 7. Poor Glycaemic Control\n", + "\n", + "The original R cohort study defined poor glycaemic control as HbA1c ≥ 7 % (LOINC 4548-4 and variants), with at least one of:\n", + "- A value ≥ 9 %, **or**\n", + "- Two consecutive values ≤ 183 days apart\n", + "\n", + "The Python translation uses `groupby` + `diff` instead of the R `aggregate(difftime(...))` loop." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d3e4f501", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Poor-control HbA1c events : 4\n", + "Patients flagged : 3\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_ideventdatevaluedays_since_lastpoor_control
1P00042014-01-1510.0NaNTrue
4P00062015-01-317.3143.0True
7P00102014-11-2912.0NaNTrue
8P00102014-12-317.232.0True
\n", + "
" + ], + "text/plain": [ + " patient_id eventdate value days_since_last poor_control\n", + "1 P0004 2014-01-15 10.0 NaN True\n", + "4 P0006 2015-01-31 7.3 143.0 True\n", + "7 P0010 2014-11-29 12.0 NaN True\n", + "8 P0010 2014-12-31 7.2 32.0 True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a1c = (\n", + " hba1c_labs\n", + " .dropna(subset=[\"value\"])\n", + " .query(\"value >= 7\")\n", + " .sort_values([\"patient_id\", \"eventdate\"])\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "# Days between consecutive HbA1c measurements per patient\n", + "a1c[\"prev_date\"] = a1c.groupby(\"patient_id\")[\"eventdate\"].shift(1)\n", + "a1c[\"days_since_last\"] = (a1c[\"eventdate\"] - a1c[\"prev_date\"]).dt.days\n", + "\n", + "# Poor control: HbA1c ≥ 9 OR interval ≤ 183 days\n", + "a1c[\"poor_control\"] = (a1c[\"value\"] >= 9) | (a1c[\"days_since_last\"] <= 183)\n", + "\n", + "poor_ctrl = a1c[a1c[\"poor_control\"]]\n", + "print(f\"Poor-control HbA1c events : {len(poor_ctrl)}\")\n", + "print(f\"Patients flagged : {poor_ctrl['patient_id'].nunique()}\")\n", + "poor_ctrl[[\"patient_id\", \"eventdate\", \"value\", \"days_since_last\", \"poor_control\"]].head(8)" + ] + }, + { + "cell_type": "markdown", + "id": "e3f4a501", + "metadata": {}, + "source": [ + "## 8. Per-Patient LOINC Coverage\n", + "\n", + "A useful QC check is verifying which patients have observations for each concept category." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "f3a4b501", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Concept coverage per patient:\n", + " hba1c weight height bmi\n", + "patient_id \n", + "P0002 True True False True\n", + "P0003 True True True False\n", + "P0004 True True True False\n", + "P0005 True True True True\n", + "P0006 True True False True\n", + "P0007 True True True False\n", + "P0008 True True False True\n", + "P0010 True True True False\n", + "P0001 False False True True\n", + "\n", + "Overall coverage rates:\n", + "hba1c 0.889\n", + "weight 0.889\n", + "height 0.667\n", + "bmi 0.556\n", + "dtype: float64\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/cz/cdvvmc8d6mn_kk380srmqfl00000gn/T/ipykernel_72177/3138325945.py:15: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", + " ], axis=1).fillna(False)\n" + ] + } + ], + "source": [ + "def concept_flag(df, loinc_list, concept_name):\n", + " return (\n", + " df[df[\"loinc\"].isin(loinc_list)]\n", + " .groupby(\"patient_id\")\n", + " .size()\n", + " .rename(concept_name)\n", + " .gt(0) # boolean: has any measurement\n", + " )\n", + "\n", + "coverage = pd.concat([\n", + " concept_flag(labs, HBA1C_LOINCS, \"hba1c\"),\n", + " concept_flag(labs, WEIGHT_LOINCS, \"weight\"),\n", + " concept_flag(labs, HEIGHT_LOINCS, \"height\"),\n", + " concept_flag(labs, BMI_LOINCS, \"bmi\"),\n", + "], axis=1).fillna(False)\n", + "\n", + "print(\"Concept coverage per patient:\")\n", + "print(coverage)\n", + "print()\n", + "print(\"Overall coverage rates:\")\n", + "print(coverage.mean().round(3))" + ] + }, + { + "cell_type": "markdown", + "id": "a4b5c601", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Task | Python API |\n", + "|---|---|\n", + "| Search LOINC by name | `df[\"long_common_name\"].str.contains(pattern, case=False)` |\n", + "| Load LOINC vocab | `loinc.load_loinc_map(\"loinc_map.csv\")` |\n", + "| Build canonical lab table | `lced.build_labtest(v_observation, v_lab_results)` |\n", + "| Filter by concept | `labs[labs[\"loinc\"].isin(HBA1C_LOINCS)]` |\n", + "| Poor glycaemic control flag | `groupby` + `shift` + `diff` on sorted dates |\n", + "\n", + "**Next steps**: See `source_lced_cohort.ipynb` for a complete T2D second-line therapy cohort study that uses these HbA1c filters." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/ehrdata/io/__init__.py b/src/ehrdata/io/__init__.py index 1ad58e7e..0062c6d6 100644 --- a/src/ehrdata/io/__init__.py +++ b/src/ehrdata/io/__init__.py @@ -1,4 +1,4 @@ -from . import omop +from . import omop, source from .csv import read_csv from .h5ad import read_h5ad, write_h5ad from .pandas import from_pandas, to_pandas diff --git a/src/ehrdata/io/source/__init__.py b/src/ehrdata/io/source/__init__.py new file mode 100644 index 00000000..4009a7e5 --- /dev/null +++ b/src/ehrdata/io/source/__init__.py @@ -0,0 +1,30 @@ +"""Non-OMOP source ingestion layer. + +Provides canonical schema definitions, generic extraction helpers, and +normalization pipelines for claims and EHR data sources (MarketScan, LCED, +CPRD). Adapters for individual data sources live under +:mod:`ehrdata.io.source.adapters`. + +Typical usage:: + + from ehrdata.io import source + + df = source.extract.union_tables([df1, df2]) + df = source.normalize.normalize_diagnosis(df) + errors = source.schema.DIAGNOSIS.validate(df) +""" + +from . import adapters, extract, normalize, schema, vocab +from .schema import ALL_SCHEMAS, TableSchema +from .to_ehrdata import to_ehrdata + +__all__ = [ + "ALL_SCHEMAS", + "TableSchema", + "adapters", + "extract", + "normalize", + "schema", + "to_ehrdata", + "vocab", +] diff --git a/src/ehrdata/io/source/adapters/__init__.py b/src/ehrdata/io/source/adapters/__init__.py new file mode 100644 index 00000000..cf8f9fff --- /dev/null +++ b/src/ehrdata/io/source/adapters/__init__.py @@ -0,0 +1,9 @@ +"""Source-specific adapter modules. + +Each adapter translates one data source's raw table shapes into canonical +DataFrames validated against :mod:`ehrdata.io.source.schema`. +""" + +from . import cprd, lced, marketscan + +__all__ = ["cprd", "lced", "marketscan"] diff --git a/src/ehrdata/io/source/adapters/cprd.py b/src/ehrdata/io/source/adapters/cprd.py new file mode 100644 index 00000000..c641db39 --- /dev/null +++ b/src/ehrdata/io/source/adapters/cprd.py @@ -0,0 +1,254 @@ +"""CPRD (Clinical Practice Research Datalink) adapter. + +Translates raw CPRD source tables (supplied as :class:`~pandas.DataFrame` +objects) into canonical DataFrames conforming to the schemas in +:mod:`ehrdata.io.source.schema`. + +CPRD differs from MarketScan and LCED in several important ways: + +1. Diagnoses are coded with **Read codes** (a UK-specific hierarchy), not ICD. + The ``dxver`` field is therefore always ``None`` for CPRD outputs. +2. Drugs are identified by **prodcode** (CPRD product dictionary) rather than + NDC or RxCUI. Drug substance names come from ``product.txt`` / ``product.csv``. +3. Lab tests are stored across two complementary file types: + + - **Clinical** + **Additional** files joined on ``(patid, adid, enttype)`` — + the Additional file holds the actual numeric test values. + - **Test** files that include both ``eventdate`` and data columns directly. + +4. CPRD dates are formatted ``DD/MM/YYYY``, not ``YYYY-MM-DD``. +5. There is no insurance or provider table in the original CPRD ETL. + +Source file reference (original CPRD GOLD extract layout): + +- ``Clinical``: patid, eventdate, medcode, enttype, adid, constype, consid +- ``Referral``: patid, eventdate, medcode, constype, consid, ... +- ``Test``: patid, eventdate, medcode, enttype, data1-data7, constype, consid +- ``Additional``: patid, enttype, adid, data1-data7 +- ``Therapy``: patid, eventdate, prodcode, bnfcode, qty, issueseq +- ``Patient``: patid, dobyr, sex, pracid +- ``Practice``: pracid, region, lcd, uts +""" + +from __future__ import annotations + +import pandas as pd + +from ehrdata.io.source.extract import union_tables +from ehrdata.io.source.normalize import ( + coerce_date, + coerce_patient_id, + deduplicate, + sort_events, +) + +_PID = "patient_id" +_CPRD_DATE_FMT = "%d/%m/%Y" + + +# --------------------------------------------------------------------------- +# Diagnosis +# --------------------------------------------------------------------------- + + +def build_diagnosis( + clinical: pd.DataFrame, + referral: pd.DataFrame, + test_data: pd.DataFrame, + *, + medical_map: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Build canonical DIAGNOSIS from CPRD clinical, referral, and test files. + + Each source contributes ``(patid, eventdate, medcode)``. When + *medical_map* is provided the medcode is translated to its Read code and + stored in ``dx``; otherwise the raw medcode string is used. + + CPRD does not use ICD coding so ``dxver`` is always ``None``. + + Args: + clinical: Clinical file rows (must include patid, eventdate, medcode). + referral: Referral file rows (must include patid, eventdate, medcode). + test_data: Test file rows (must include patid, eventdate, medcode). + medical_map: Optional medcode→readcode DataFrame from + :func:`~ehrdata.io.source.vocab.readcode.load_medical_map`. + + Returns: + Canonical DIAGNOSIS DataFrame sorted by patient then eventdate. + """ + parts = [] + for src in (clinical, referral, test_data): + p = src[["patid", "eventdate", "medcode"]].copy() + p["patid"] = coerce_patient_id(p["patid"]) + p["eventdate"] = coerce_date(p["eventdate"], formats=[_CPRD_DATE_FMT]) + if medical_map is not None: + map_ = medical_map[["medcode", "readcode"]].drop_duplicates(subset=["medcode"]) + p["medcode"] = p["medcode"].astype(str) + p = p.merge(map_, on="medcode", how="left") + p["dx"] = p["readcode"].astype(object) + p = p.drop(columns=["readcode", "medcode"]) + else: + p["dx"] = p["medcode"].astype(str) + p = p.drop(columns=["medcode"]) + p = p.rename(columns={"patid": _PID}) + p = p.dropna(subset=["dx"]) + parts.append(p[[_PID, "eventdate", "dx"]]) + + out = union_tables(parts) + out["dxver"] = pd.array([None] * len(out), dtype=object) + out = out[[_PID, "dxver", "eventdate", "dx"]] + return deduplicate(sort_events(out, patient_col=_PID, date_col="eventdate")) + + +# --------------------------------------------------------------------------- +# Therapy +# --------------------------------------------------------------------------- + + +def build_therapy( + therapy_data: pd.DataFrame, + *, + product_map: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Build canonical THERAPY from CPRD therapy files. + + ``eventdate`` is mapped to ``fill_date`` (the prescription/dispensing date). + ``prescription_date``, ``start_date``, and ``end_date`` are not available in + CPRD and are set to ``NaT``. ``ndc11`` and ``rxcui`` are ``None``. + + When *product_map* is provided, prodcode is joined to obtain the drug + substance name stored in ``ingredient``. + + Args: + therapy_data: Therapy file rows (patid, eventdate, prodcode, ...). + product_map: Optional prodcode→drugsubstance DataFrame from + :func:`~ehrdata.io.source.vocab.prodcode.load_product_map`. + + Returns: + Canonical THERAPY DataFrame. + """ + out = therapy_data[["patid", "eventdate", "prodcode"]].copy() + out["patid"] = coerce_patient_id(out["patid"]) + out["fill_date"] = coerce_date(out["eventdate"], formats=[_CPRD_DATE_FMT]) + out = out.drop(columns=["eventdate"]) + + if product_map is not None: + map_ = product_map[["prodcode", "drugsubstance"]].drop_duplicates(subset=["prodcode"]) + out = out.merge(map_, on="prodcode", how="left") + out["ingredient"] = out["drugsubstance"].astype(object) + out = out.drop(columns=["drugsubstance"]) + else: + out["ingredient"] = pd.array([None] * len(out), dtype=object) + + out = out.drop(columns=["prodcode"]) + out = out.rename(columns={"patid": _PID}) + + for col in ("prescription_date", "start_date", "end_date"): + out[col] = pd.NaT + out["refill"] = pd.array([pd.NA] * len(out), dtype="Int64") + out["rxcui"] = pd.array([None] * len(out), dtype=object) + out["ndc11"] = pd.array([None] * len(out), dtype=object) + + out = out[[_PID, "prescription_date", "start_date", "fill_date", "end_date", + "refill", "rxcui", "ndc11", "ingredient"]] + return deduplicate(out) + + +# --------------------------------------------------------------------------- +# Lab test +# --------------------------------------------------------------------------- + + +def build_labtest( + clinical: pd.DataFrame, + additional: pd.DataFrame, + test_data: pd.DataFrame, + *, + entity_enttypes: set[int] | None = None, +) -> pd.DataFrame: + """Build canonical LABTEST from CPRD clinical+additional and test files. + + Two complementary sources are combined: + + 1. **Clinical** (patid, eventdate, enttype, adid) inner-joined with + **Additional** (patid, enttype, adid, data2, data3, data4) on the + composite key ``(patid, enttype, adid)`` — replicates the + ``merge(..., by=c("patid","adid","enttype"))`` in the original R ETL. + 2. **Test** files (patid, eventdate, enttype, data2, data3, data4) that + already include both date and data columns. + + Column mapping: ``data2`` → ``value``, ``data3`` → ``unit``, + ``data4`` → ``valuecat``. CPRD does not provide LOINC codes so ``loinc`` + is always ``None``. + + Args: + clinical: Clinical file rows (patid, eventdate, enttype, adid). + additional: Additional file rows (patid, enttype, adid, data2, data3, data4). + test_data: Test file rows (patid, eventdate, enttype, data2, data3, data4). + entity_enttypes: Optional set of ``enttype`` integer codes to retain; + all entity types are included when ``None``. + + Returns: + Canonical LABTEST DataFrame sorted by patient then eventdate. + """ + # Part 1: clinical + additional inner-join to recover eventdate for the data rows + clin_cols = [c for c in ("patid", "eventdate", "enttype", "adid") if c in clinical.columns] + addi_cols = [c for c in ("patid", "enttype", "adid", "data2", "data3", "data4") if c in additional.columns] + clin_addi = ( + clinical[clin_cols] + .merge(additional[addi_cols], on=["patid", "enttype", "adid"], how="inner") + .drop(columns=["adid"]) + ) + + # Part 2: test file already has all needed columns + test_cols = [c for c in ("patid", "eventdate", "enttype", "data2", "data3", "data4") if c in test_data.columns] + test_part = test_data[test_cols].copy() + + out = pd.concat([clin_addi, test_part], ignore_index=True) + + if entity_enttypes is not None: + out = out[out["enttype"].isin(entity_enttypes)] + + out = out.rename(columns={ + "patid": _PID, + "data2": "value", + "data3": "unit", + "data4": "valuecat", + }) + out[_PID] = coerce_patient_id(out[_PID]) + out["eventdate"] = coerce_date(out["eventdate"], formats=[_CPRD_DATE_FMT]) + out["loinc"] = pd.array([None] * len(out), dtype=object) + + out = out[[_PID, "eventdate", "value", "valuecat", "unit", "loinc"]] + return deduplicate(sort_events(out, patient_col=_PID, date_col="eventdate")) + + +# --------------------------------------------------------------------------- +# Patient information +# --------------------------------------------------------------------------- + + +def build_patinfo(*patient_tables: pd.DataFrame) -> pd.DataFrame: + """Build canonical PATINFO from one or more CPRD patient extract files. + + Each table must contain at least ``patid``, ``dobyr``, and ``sex``. + Rows are unioned across all tables and duplicates removed. + + Args: + *patient_tables: One or more patient DataFrames. + + Returns: + Canonical PATINFO DataFrame (patient_id, dobyr, sex). + """ + parts = [] + for pt in patient_tables: + dob_col = "dobyr" if "dobyr" in pt.columns else "yob" + p = pt[["patid", dob_col, "sex"]].copy() + p = p.rename(columns={"patid": _PID, dob_col: "dobyr"}) + p[_PID] = coerce_patient_id(p[_PID]) + parts.append(p) + + out = union_tables(parts) + out["dobyr"] = pd.array(pd.to_numeric(out["dobyr"], errors="coerce"), dtype="Int64") + out["sex"] = out["sex"].astype(object) + return deduplicate(out[[_PID, "dobyr", "sex"]]) diff --git a/src/ehrdata/io/source/adapters/lced.py b/src/ehrdata/io/source/adapters/lced.py new file mode 100644 index 00000000..0ca90d5e --- /dev/null +++ b/src/ehrdata/io/source/adapters/lced.py @@ -0,0 +1,476 @@ +"""IBM LCED (Limited Claims-EMR Data) adapter. + +Translates raw IBM LCED source tables (supplied as :class:`~pandas.DataFrame` +objects) into canonical DataFrames conforming to the schemas in +:mod:`ehrdata.io.source.schema`. + +LCED differs from MarketScan in three important ways: + +1. The patient identifier is already named ``patient_id`` in every source view + (no ``enrolid`` rename needed). +2. Therapy comes from **two** independent sources that must be joined separately + before unioning — ``v_drug`` (RxCUI-based, prescription records) and + ``v_outpatient_drug_claims`` (NDC-based, pharmacy claims). +3. Lab tests are present in two sources — ``v_observation`` (structured EMR + observations) and ``v_lab_results`` (claims-based lab results). +4. An extra ``habit`` table captures lifestyle/survey data (smoking, BMI, etc.) + obtained by joining ``v_habit`` with ``v_encounter``. + +Source view reference (original LCED PostgreSQL schema ``lced.*``): + +- ``v_facility_header``: facility encounters +- ``v_inpatient_admissions``: inpatient admissions +- ``v_inpatient_services``: inpatient service lines +- ``v_lab_results``: lab result claims +- ``v_outpatient_services``: outpatient service lines +- ``v_drug``: EMR prescription records (RxCUI) +- ``v_outpatient_drug_claims``: pharmacy claims (NDC) +- ``v_observation``: structured EMR lab observations (LOINC) +- ``v_habit``: patient lifestyle/survey responses +- ``v_encounter``: encounter dates for habit join +- ``v_annual_summary_enrollment``: annual enrollment snapshots +- ``v_detail_enrollment``: enrollment coverage detail with plan info +""" + +from __future__ import annotations + +import pandas as pd + +from ehrdata.io.source.extract import union_tables, unnest_codes +from ehrdata.io.source.normalize import ( + coerce_date, + coerce_patient_id, + deduplicate, + infer_icd_version, + sort_events, +) + +_PID = "patient_id" + + +# --------------------------------------------------------------------------- +# Diagnosis +# --------------------------------------------------------------------------- + + +def build_diagnosis( + facility_header: pd.DataFrame, + inpatient_admissions: pd.DataFrame, + inpatient_services: pd.DataFrame, + lab_results: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical diagnosis table from five LCED sources. + + Mirrors ``LCED Data Cleaning.R`` lines 30–54. Identical in structure to + the MarketScan adapter except that ``patient_id`` is already the correct + column name and a fifth source (``v_lab_results``) contributes ``dx1``. + + Args: + facility_header: Must contain ``patient_id``, ``dxver``, ``svcdate``, + ``dx1``–``dx9``. + inpatient_admissions: Must contain ``patient_id``, ``dxver``, + ``admdate``, ``pdx``, ``dx1``–``dx15``. + inpatient_services: Must contain ``patient_id``, ``dxver``, + ``svcdate``, ``pdx``, ``dx1``–``dx4``. + lab_results: Must contain ``patient_id``, ``dxver``, ``svcdate``, + ``dx1``. + outpatient_services: Must contain ``patient_id``, ``dxver``, + ``svcdate``, ``dx1``–``dx4``. + + Returns: + Canonical diagnosis DataFrame conforming to + :data:`~ehrdata.io.source.schema.DIAGNOSIS`. + """ + parts = [ + _unnest_dx(facility_header, date_col="svcdate", + dx_cols=[f"dx{i}" for i in range(1, 10)]), + _unnest_dx(inpatient_admissions, date_col="admdate", + dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), + _unnest_dx(inpatient_services, date_col="svcdate", + dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), + _unnest_dx(lab_results, date_col="svcdate", dx_cols=["dx1"]), + _unnest_dx(outpatient_services, date_col="svcdate", + dx_cols=[f"dx{i}" for i in range(1, 5)]), + ] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = infer_icd_version(df) + df = deduplicate(df) + df = sort_events(df) + return df + + +def _unnest_dx(src: pd.DataFrame, *, date_col: str, dx_cols: list[str]) -> pd.DataFrame: + """Select and unnest diagnosis codes from one LCED source table.""" + present_dx = [c for c in dx_cols if c in src.columns] + dxver_cols = ["dxver"] if "dxver" in src.columns else [] + tmp = src[[_PID, date_col] + dxver_cols + present_dx].copy() + tmp = tmp.rename(columns={date_col: "eventdate"}) + if "dxver" not in tmp.columns: + tmp["dxver"] = None + return unnest_codes(tmp, id_cols=[_PID, "eventdate", "dxver"], code_cols=present_dx, value_name="dx") + + +# --------------------------------------------------------------------------- +# Therapy +# --------------------------------------------------------------------------- + + +def build_therapy( + v_drug: pd.DataFrame, + v_outpatient_drug_claims: pd.DataFrame, + *, + ndc_map: pd.DataFrame | None = None, + rxcui_map: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Build the canonical therapy table from two LCED drug sources. + + Mirrors ``LCED Data Cleaning.R`` lines 68–88. + + ``v_drug`` holds EMR prescriptions keyed by RxCUI; ``v_outpatient_drug_claims`` + holds pharmacy claims keyed by NDC. Each source is ingredient-joined with + its respective vocabulary before the two are unioned. + + Args: + v_drug: ``lced.v_drug`` view data. Must contain ``patient_id``, + ``prescription_date``, ``start_date``, ``end_date``, ``rx_cui``. + v_outpatient_drug_claims: ``lced.v_outpatient_drug_claims`` view data. + Must contain ``patient_id``, ``svcdate``, ``daysupp``, ``refill``, + ``ndcnum``. + ndc_map: Optional NDC→ingredient map (from + :func:`~ehrdata.io.source.vocab.ndc.load_ndc_ingredient_map`). + rxcui_map: Optional RxCUI→ingredient map (from + :func:`~ehrdata.io.source.vocab.rxnorm.load_rxcui_ingredient_map`). + + Returns: + Canonical therapy DataFrame conforming to + :data:`~ehrdata.io.source.schema.THERAPY`. + """ + drug_part = _build_drug_part(v_drug, rxcui_map=rxcui_map) + claims_part = _build_claims_part(v_outpatient_drug_claims, ndc_map=ndc_map) + df = union_tables([drug_part, claims_part]) + df = deduplicate(df) + return df[["patient_id", "prescription_date", "start_date", "fill_date", + "end_date", "refill", "rxcui", "ndc11", "ingredient"]] + + +def _build_drug_part(src: pd.DataFrame, *, rxcui_map: pd.DataFrame | None) -> pd.DataFrame: + """Build the RxCUI-based (v_drug) contribution to therapy.""" + df = pd.DataFrame() + df[_PID] = coerce_patient_id(src[_PID]) + df["prescription_date"] = coerce_date(src.get("prescription_date")) + df["start_date"] = coerce_date(src.get("start_date")) + df["fill_date"] = pd.NaT + df["end_date"] = coerce_date(src.get("end_date")) + df["refill"] = pd.array([pd.NA] * len(src), dtype="Int64") + df["rxcui"] = src["rx_cui"].astype(str).str.strip() if "rx_cui" in src.columns else None + df["ndc11"] = None + if rxcui_map is not None: + from ehrdata.io.source.vocab.rxnorm import join_ingredient_by_rxcui + df = join_ingredient_by_rxcui(df, rxcui_map) + else: + df["ingredient"] = None + return df + + +def _build_claims_part(src: pd.DataFrame, *, ndc_map: pd.DataFrame | None) -> pd.DataFrame: + """Build the NDC-based (v_outpatient_drug_claims) contribution to therapy.""" + df = pd.DataFrame() + df[_PID] = coerce_patient_id(src[_PID]) + df["prescription_date"] = pd.NaT + df["start_date"] = pd.NaT + df["fill_date"] = coerce_date(src["svcdate"]) + daysupp = pd.to_numeric(src["daysupp"], errors="coerce") + df["end_date"] = df["fill_date"] + pd.to_timedelta(daysupp, unit="D") + df["refill"] = pd.to_numeric(src["refill"], errors="coerce").astype("Int64") + df["rxcui"] = None + df["ndc11"] = src["ndcnum"].astype(str).str.strip().str.zfill(11) + if ndc_map is not None: + from ehrdata.io.source.vocab.ndc import join_ingredient_by_ndc + df = join_ingredient_by_ndc(df, ndc_map) + else: + df["ingredient"] = None + return df + + +# --------------------------------------------------------------------------- +# Lab tests +# --------------------------------------------------------------------------- + + +def build_labtest( + v_observation: pd.DataFrame, + v_lab_results: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical lab-test table from two LCED sources. + + Mirrors ``LCED Data Cleaning.R`` lines 93–98. + + ``v_observation`` provides structured EMR observations (``std_value`` / + ``std_uom`` / ``loinc_test_id``); ``v_lab_results`` provides claims-based + results (``result`` / ``resltcat`` / ``resunit`` / ``loinccd``). + + Args: + v_observation: ``lced.v_observation`` view data. Must contain + ``patient_id`` and ``observation_date``. Optional: + ``std_value``, ``std_uom``, ``loinc_test_id``. + v_lab_results: ``lced.v_lab_results`` view data. Must contain + ``patient_id`` and ``svcdate``. Optional: ``result``, + ``resltcat``, ``resunit``, ``loinccd``. + + Returns: + Canonical labtest DataFrame conforming to + :data:`~ehrdata.io.source.schema.LABTEST`. + """ + obs_part = _build_observation_part(v_observation) + lab_part = _build_lab_results_part(v_lab_results) + df = union_tables([obs_part, lab_part]) + df[_PID] = coerce_patient_id(df[_PID]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = deduplicate(df) + df = sort_events(df) + return df + + +def _build_observation_part(src: pd.DataFrame) -> pd.DataFrame: + """Map v_observation columns to canonical labtest schema.""" + df = pd.DataFrame() + df[_PID] = src[_PID] + df["eventdate"] = src.get("observation_date") + df["value"] = src.get("std_value", pd.Series(dtype=object, index=src.index)) + df["valuecat"] = None + df["unit"] = src.get("std_uom", pd.Series(dtype=object, index=src.index)) + df["loinc"] = src.get("loinc_test_id", pd.Series(dtype=object, index=src.index)) + return df + + +def _build_lab_results_part(src: pd.DataFrame) -> pd.DataFrame: + """Map v_lab_results columns to canonical labtest schema.""" + df = pd.DataFrame() + df[_PID] = src[_PID] + df["eventdate"] = src.get("svcdate") + df["value"] = src.get("result", pd.Series(dtype=object, index=src.index)).astype(str).where( + src.get("result", pd.Series(dtype=object, index=src.index)).notna(), None + ) + df["valuecat"] = src.get("resltcat", pd.Series(dtype=object, index=src.index)) + df["unit"] = src.get("resunit", pd.Series(dtype=object, index=src.index)) + df["loinc"] = src.get("loinccd", pd.Series(dtype=object, index=src.index)) + return df + + +# --------------------------------------------------------------------------- +# Procedure +# --------------------------------------------------------------------------- + + +def build_procedure( + facility_header: pd.DataFrame, + inpatient_admissions: pd.DataFrame, + inpatient_services: pd.DataFrame, + lab_results: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical procedure table from five LCED sources. + + Mirrors ``LCED Data Cleaning.R`` lines 110–133. Identical in structure to + MarketScan except ``patient_id`` is already correct and ``v_lab_results`` + contributes an additional procedure source. + + Note: ``inpatient_services`` unnests ``array[pdx, proc1]``, replicating + the original ETL which included the primary diagnosis alongside the + procedure code. + + Args: + facility_header: Must contain ``patient_id``, ``svcdate``, + ``proc1``–``proc6``. + inpatient_admissions: Must contain ``patient_id``, ``admdate``, + ``pproc``, ``proc1``–``proc15``. + inpatient_services: Must contain ``patient_id``, ``svcdate``, + ``proctyp``, ``pdx``, ``proc1``. + lab_results: Must contain ``patient_id``, ``svcdate``, ``proctyp``, + ``proc1``. + outpatient_services: Must contain ``patient_id``, ``svcdate``, + ``proctyp``, ``proc1``. + + Returns: + Canonical procedure DataFrame conforming to + :data:`~ehrdata.io.source.schema.PROCEDURE`. + """ + parts = [ + _unnest_proc(facility_header, date_col="svcdate", + proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None), + _unnest_proc(inpatient_admissions, date_col="admdate", + proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], proctype_col=None), + _unnest_proc(inpatient_services, date_col="svcdate", + proc_cols=["pdx", "proc1"], proctype_col="proctyp"), + _unnest_proc(lab_results, date_col="svcdate", + proc_cols=["proc1"], proctype_col="proctyp"), + _unnest_proc(outpatient_services, date_col="svcdate", + proc_cols=["proc1"], proctype_col="proctyp"), + ] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = deduplicate(df) + df = sort_events(df) + return df + + +def _unnest_proc( + src: pd.DataFrame, + *, + date_col: str, + proc_cols: list[str], + proctype_col: str | None, +) -> pd.DataFrame: + """Select and unnest procedure codes from one LCED source table.""" + present_proc = [c for c in proc_cols if c in src.columns] + tmp = src[[_PID, date_col] + present_proc].copy() + tmp = tmp.rename(columns={date_col: "eventdate"}) + if proctype_col and proctype_col in src.columns: + tmp["proctype"] = src[proctype_col].values + else: + tmp["proctype"] = None + return unnest_codes(tmp, id_cols=[_PID, "eventdate", "proctype"], code_cols=present_proc, value_name="proc") + + +# --------------------------------------------------------------------------- +# Habit +# --------------------------------------------------------------------------- + + +def build_habit( + v_habit: pd.DataFrame, + v_encounter: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical habit table from LCED survey/lifestyle data. + + Mirrors ``LCED Data Cleaning.R`` lines 148–155. + + Joins ``v_habit`` (patient lifestyle responses) with ``v_encounter`` + (encounter dates) on ``encounter_join_id``, then drops rows where + ``mapped_question_answer`` is null and removes the join key column. + + Args: + v_habit: ``lced.v_habit`` view data. Must contain ``patient_id``, + ``mapped_question_answer``, and ``encounter_join_id``. + v_encounter: ``lced.v_encounter`` view data. Must contain + ``encounter_date`` and ``encounter_join_id``. + + Returns: + Canonical habit DataFrame conforming to + :data:`~ehrdata.io.source.schema.HABIT`. + """ + df = v_habit.merge( + v_encounter[["encounter_join_id", "encounter_date"]], + on="encounter_join_id", + how="left", + ) + df = df[df["mapped_question_answer"].notna()].copy() + df[_PID] = coerce_patient_id(df[_PID]) + df["encounter_date"] = coerce_date(df["encounter_date"]) + df = df.drop(columns="encounter_join_id") + df = deduplicate(df) + df = df.sort_values([_PID, "encounter_date", "mapped_question_answer"], + na_position="last").reset_index(drop=True) + return df[[_PID, "encounter_date", "mapped_question_answer"]] + + +# --------------------------------------------------------------------------- +# Patinfo +# --------------------------------------------------------------------------- + + +def build_patinfo(*source_tables: pd.DataFrame) -> pd.DataFrame: + """Build the canonical patient-info table by unioning LCED sources. + + LCED patinfo only extracts the three canonical columns + (``patient_id``, ``dobyr``, ``sex``) — unlike MarketScan, no extra + regional/plan columns are available at this level. + + Mirrors the seven-source union + ``distinct_all()`` in + ``LCED Data Cleaning.R`` lines 166–182. + + Args: + *source_tables: One or more DataFrames each containing at least + ``patient_id``, ``dobyr``, and ``sex``. + + Returns: + Canonical patinfo DataFrame with columns ``patient_id``, ``dobyr``, + ``sex``. + """ + parts = [t[[c for c in ["patient_id", "dobyr", "sex"] if c in t.columns]].copy() + for t in source_tables] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["dobyr"] = pd.to_numeric(df["dobyr"], errors="coerce").astype("Int64") + return deduplicate(df) + + +# --------------------------------------------------------------------------- +# Insurance +# --------------------------------------------------------------------------- + + +def build_insurance( + facility_header: pd.DataFrame, + inpatient_services: pd.DataFrame, + outpatient_drug_claims: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical insurance table from four LCED sources. + + Mirrors ``LCED Data Cleaning.R`` lines 193–202. + + Args: + facility_header: Must contain ``patient_id``, ``svcdate``, ``cob``, + ``coins``, ``copay``. + inpatient_services: Same columns. + outpatient_drug_claims: Same columns. + outpatient_services: Same columns. + + Returns: + Canonical insurance DataFrame conforming to + :data:`~ehrdata.io.source.schema.INSURANCE`. + """ + _COLS = ["patient_id", "svcdate", "cob", "coins", "copay"] + parts = [src[[c for c in _COLS if c in src.columns]].copy() + for src in (facility_header, inpatient_services, outpatient_drug_claims, outpatient_services)] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["svcdate"] = coerce_date(df["svcdate"]) + for col in ("cob", "coins", "copay"): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + return deduplicate(df) + + +# --------------------------------------------------------------------------- +# Provider +# --------------------------------------------------------------------------- + + +def build_provider(v_detail_enrollment: pd.DataFrame) -> pd.DataFrame: + """Build the canonical provider table from LCED enrollment detail. + + Mirrors ``LCED Data Cleaning.R`` lines 214–221. + + Args: + v_detail_enrollment: ``lced.v_detail_enrollment`` view data. Must + contain ``patient_id``. Optional: ``dtstart``, ``dtend``, + ``plantyp``, ``rx``, ``hlthplan``. + + Returns: + Canonical provider DataFrame conforming to + :data:`~ehrdata.io.source.schema.PROVIDER`. + """ + _COLS = ["patient_id", "dtstart", "dtend", "plantyp", "rx", "hlthplan"] + present = [c for c in _COLS if c in v_detail_enrollment.columns] + df = v_detail_enrollment[present].copy() + df[_PID] = coerce_patient_id(df[_PID]) + for col in ("dtstart", "dtend"): + if col in df.columns: + df[col] = coerce_date(df[col]) + return deduplicate(df) diff --git a/src/ehrdata/io/source/adapters/marketscan.py b/src/ehrdata/io/source/adapters/marketscan.py new file mode 100644 index 00000000..e6e4a631 --- /dev/null +++ b/src/ehrdata/io/source/adapters/marketscan.py @@ -0,0 +1,353 @@ +"""IBM MarketScan claims adapter. + +Translates raw IBM MarketScan Commercial Claims source tables (supplied as +:class:`~pandas.DataFrame` objects) into canonical DataFrames conforming to +the schemas defined in :mod:`ehrdata.io.source.schema`. + +Each ``build_*`` function accepts the specific source tables it needs, applies +column selection / renaming, unnests wide code arrays into long format, unions +multiple sources, and delegates final normalization to +:mod:`ehrdata.io.source.normalize`. + +Source column reference (original MarketScan PostgreSQL view names): + +- ``facility_header``: facility/outpatient encounter header +- ``inpatient_admissions``: inpatient admission records +- ``inpatient_services``: inpatient line-item services +- ``outpatient_services``: outpatient line-item services +- ``outpatient_prescription_drugs``: pharmacy claims +- ``enrollment_annual_summary``: annual enrollment snapshot +- ``enrollment_detail``: enrollment coverage detail with plan info +""" + +from __future__ import annotations + +import pandas as pd + +from ehrdata.io.source.extract import union_tables, unnest_codes +from ehrdata.io.source.normalize import ( + coerce_date, + coerce_patient_id, + deduplicate, + infer_icd_version, + sort_events, +) + +# --------------------------------------------------------------------------- +# Column name constants +# --------------------------------------------------------------------------- + +# Canonical patient ID (MarketScan calls it enrolid) +_PID = "patient_id" +_ENROLID = "enrolid" + +# Patinfo columns that exist in MarketScan beyond the canonical three +_PATINFO_EXTRA_COLS = [ + "efamid", "year", "region", "msa", "wgtkey", + "eeclass", "eestatu", "egeoloc", "emprel", "indstry", +] + +# All patinfo columns as they appear in MarketScan source tables +_PATINFO_SRC_COLS = [_ENROLID, "dobyr", "sex"] + _PATINFO_EXTRA_COLS + + +# --------------------------------------------------------------------------- +# Diagnosis +# --------------------------------------------------------------------------- + + +def build_diagnosis( + facility_header: pd.DataFrame, + inpatient_admissions: pd.DataFrame, + inpatient_services: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical diagnosis table from four MarketScan sources. + + Mirrors the ``unnest(array[dx1..dx9])`` + ``union_all`` + ``distinct`` + pattern from ``MarketScan Data Cleaning.R`` (lines 44–56). + + Args: + facility_header: ``commercial.facility_header`` view data. + Must contain ``enrolid``, ``dxver``, ``svcdate``, ``dx1``–``dx9``. + inpatient_admissions: ``commercial.inpatient_admissions`` view data. + Must contain ``enrolid``, ``dxver``, ``admdate``, ``pdx``, + ``dx1``–``dx15``. + inpatient_services: ``commercial.inpatient_services`` view data. + Must contain ``enrolid``, ``dxver``, ``svcdate``, ``pdx``, + ``dx1``–``dx4``. + outpatient_services: ``commercial.outpatient_services`` view data. + Must contain ``enrolid``, ``dxver``, ``svcdate``, ``dx1``–``dx4``. + + Returns: + Canonical diagnosis DataFrame conforming to + :data:`~ehrdata.io.source.schema.DIAGNOSIS`. + """ + parts = [ + _unnest_dx(facility_header, date_col="svcdate", + dx_cols=[f"dx{i}" for i in range(1, 10)]), + _unnest_dx(inpatient_admissions, date_col="admdate", + dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), + _unnest_dx(inpatient_services, date_col="svcdate", + dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), + _unnest_dx(outpatient_services, date_col="svcdate", + dx_cols=[f"dx{i}" for i in range(1, 5)]), + ] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = infer_icd_version(df) + df = deduplicate(df) + df = sort_events(df) + return df + + +def _unnest_dx(src: pd.DataFrame, *, date_col: str, dx_cols: list[str]) -> pd.DataFrame: + """Select, rename, and unnest diagnosis codes from one source table.""" + present_dx = [c for c in dx_cols if c in src.columns] + dxver_col = ["dxver"] if "dxver" in src.columns else [] + tmp = src[[_ENROLID, date_col] + dxver_col + present_dx].copy() + tmp = tmp.rename(columns={_ENROLID: _PID, date_col: "eventdate"}) + if "dxver" not in tmp.columns: + tmp["dxver"] = None + return unnest_codes(tmp, id_cols=[_PID, "eventdate", "dxver"], code_cols=present_dx, value_name="dx") + + +# --------------------------------------------------------------------------- +# Therapy +# --------------------------------------------------------------------------- + + +def build_therapy( + outpatient_prescription_drugs: pd.DataFrame, + *, + ndc_map: pd.DataFrame | None = None, +) -> pd.DataFrame: + """Build the canonical therapy table from MarketScan pharmacy claims. + + Mirrors ``MarketScan Data Cleaning.R`` lines 82–98: + - Maps ``svcdate`` → ``fill_date`` + - Calculates ``end_date = fill_date + daysupp`` days + - Renames ``ndcnum`` → ``ndc11`` + - Optionally joins ingredient from NDC map + + Args: + outpatient_prescription_drugs: ``commercial.outpatient_prescription_drugs`` + view data. Must contain ``enrolid``, ``svcdate``, ``daysupp``, + ``refill``, and ``ndcnum``. + ndc_map: Optional NDC→ingredient map as returned by + :func:`~ehrdata.io.source.vocab.ndc.load_ndc_ingredient_map`. + When ``None`` the ``ingredient`` column is set to ``None``. + + Returns: + Canonical therapy DataFrame conforming to + :data:`~ehrdata.io.source.schema.THERAPY`. + """ + src = outpatient_prescription_drugs + df = pd.DataFrame() + df[_PID] = coerce_patient_id(src[_ENROLID]) + df["prescription_date"] = pd.NaT + df["start_date"] = pd.NaT + df["fill_date"] = coerce_date(src["svcdate"]) + df["refill"] = pd.to_numeric(src["refill"], errors="coerce").astype("Int64") + df["rxcui"] = None + df["ndc11"] = src["ndcnum"].astype(str).str.strip().str.zfill(11) + + # Calculate end_date = fill_date + daysupp days + daysupp = pd.to_numeric(src["daysupp"], errors="coerce") + df["end_date"] = df["fill_date"] + pd.to_timedelta(daysupp, unit="D") + + if ndc_map is not None: + from ehrdata.io.source.vocab.ndc import join_ingredient_by_ndc + df = join_ingredient_by_ndc(df, ndc_map) + else: + df["ingredient"] = None + + df = deduplicate(df) + return df[["patient_id", "prescription_date", "start_date", "fill_date", + "end_date", "refill", "rxcui", "ndc11", "ingredient"]] + + +# --------------------------------------------------------------------------- +# Procedure +# --------------------------------------------------------------------------- + + +def build_procedure( + facility_header: pd.DataFrame, + inpatient_admissions: pd.DataFrame, + inpatient_services: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical procedure table from four MarketScan sources. + + Mirrors ``MarketScan Data Cleaning.R`` lines 121–140. + + Note: ``inpatient_services`` unnests ``array[pdx, proc1]`` — this + replicates the original ETL which included the primary diagnosis code + alongside the procedure code in the inpatient services procedure array. + + Args: + facility_header: Must contain ``enrolid``, ``svcdate``, + ``proc1``–``proc6``. + inpatient_admissions: Must contain ``enrolid``, ``admdate``, + ``pproc``, ``proc1``–``proc15``. + inpatient_services: Must contain ``enrolid``, ``svcdate``, + ``proctyp``, ``pdx``, ``proc1``. + outpatient_services: Must contain ``enrolid``, ``svcdate``, + ``proctyp``, ``proc1``. + + Returns: + Canonical procedure DataFrame conforming to + :data:`~ehrdata.io.source.schema.PROCEDURE`. + """ + parts = [ + _unnest_proc(facility_header, date_col="svcdate", + proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None), + _unnest_proc(inpatient_admissions, date_col="admdate", + proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], proctype_col=None), + _unnest_proc(inpatient_services, date_col="svcdate", + proc_cols=["pdx", "proc1"], proctype_col="proctyp"), + _unnest_proc(outpatient_services, date_col="svcdate", + proc_cols=["proc1"], proctype_col="proctyp"), + ] + df = union_tables(parts) + df[_PID] = coerce_patient_id(df[_PID]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = deduplicate(df) + df = sort_events(df) + return df + + +def _unnest_proc( + src: pd.DataFrame, + *, + date_col: str, + proc_cols: list[str], + proctype_col: str | None, +) -> pd.DataFrame: + """Select, rename, and unnest procedure codes from one source table.""" + present_proc = [c for c in proc_cols if c in src.columns] + tmp = src[[_ENROLID, date_col] + present_proc].copy() + tmp = tmp.rename(columns={_ENROLID: _PID, date_col: "eventdate"}) + if proctype_col and proctype_col in src.columns: + tmp["proctype"] = src[proctype_col].values + else: + tmp["proctype"] = None + long = unnest_codes(tmp, id_cols=[_PID, "eventdate", "proctype"], code_cols=present_proc, value_name="proc") + return long + + +# --------------------------------------------------------------------------- +# Patinfo +# --------------------------------------------------------------------------- + + +def build_patinfo(*source_tables: pd.DataFrame) -> pd.DataFrame: + """Build the canonical patient-info table by unioning all MarketScan sources. + + Accepts any number of source DataFrames that contain ``enrolid``, + ``dobyr``, and ``sex`` columns. Extra MarketScan columns (``efamid``, + ``year``, ``region``, ``msa``, etc.) are carried through when present in + all inputs. + + Mirrors the seven-source INSERT + ``distinct_all()`` pattern in + ``MarketScan Data Cleaning.R`` lines 159–210. + + Args: + *source_tables: One or more source DataFrames. Each must contain at + least ``enrolid``, ``dobyr``, and ``sex``. + + Returns: + Canonical patinfo DataFrame. Always contains ``patient_id``, + ``dobyr``, ``sex``. Extra MarketScan columns are included when + present. + """ + _REQUIRED = [_ENROLID, "dobyr", "sex"] + available_extra = [ + c for c in _PATINFO_EXTRA_COLS + if all(c in t.columns for t in source_tables) + ] + keep_cols = _REQUIRED + available_extra + + parts = [t[[c for c in keep_cols if c in t.columns]].copy() for t in source_tables] + df = union_tables(parts) + df = df.rename(columns={_ENROLID: _PID}) + df[_PID] = coerce_patient_id(df[_PID]) + df["dobyr"] = pd.to_numeric(df["dobyr"], errors="coerce").astype("Int64") + df = deduplicate(df) + return df + + +# --------------------------------------------------------------------------- +# Insurance +# --------------------------------------------------------------------------- + + +def build_insurance( + facility_header: pd.DataFrame, + inpatient_services: pd.DataFrame, + outpatient_prescription_drugs: pd.DataFrame, + outpatient_services: pd.DataFrame, +) -> pd.DataFrame: + """Build the canonical insurance table from four MarketScan sources. + + Mirrors ``MarketScan Data Cleaning.R`` lines 221–250. + + Args: + facility_header: Must contain ``enrolid``, ``svcdate``, ``cob``, + ``coins``, ``copay``. + inpatient_services: Same columns. + outpatient_prescription_drugs: Same columns. + outpatient_services: Same columns. + + Returns: + Canonical insurance DataFrame conforming to + :data:`~ehrdata.io.source.schema.INSURANCE`. + """ + _INS_COLS = [_ENROLID, "svcdate", "cob", "coins", "copay"] + parts = [] + for src in (facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + present = [c for c in _INS_COLS if c in src.columns] + parts.append(src[present].copy()) + + df = union_tables(parts) + df = df.rename(columns={_ENROLID: _PID}) + df[_PID] = coerce_patient_id(df[_PID]) + df["svcdate"] = coerce_date(df["svcdate"]) + for col in ("cob", "coins", "copay"): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + df = deduplicate(df) + return df + + +# --------------------------------------------------------------------------- +# Provider +# --------------------------------------------------------------------------- + + +def build_provider(enrollment_detail: pd.DataFrame) -> pd.DataFrame: + """Build the canonical provider table from enrollment detail. + + Mirrors ``MarketScan Data Cleaning.R`` lines 256–265. + + Args: + enrollment_detail: ``commercial.enrollment_detail`` view data. + Must contain ``enrolid``. Optional columns: ``dtstart``, + ``dtend``, ``plantyp``, ``rx``, ``hlthplan``. + + Returns: + Canonical provider DataFrame conforming to + :data:`~ehrdata.io.source.schema.PROVIDER`. + """ + _PROV_COLS = [_ENROLID, "dtstart", "dtend", "plantyp", "rx", "hlthplan"] + present = [c for c in _PROV_COLS if c in enrollment_detail.columns] + df = enrollment_detail[present].copy() + df = df.rename(columns={_ENROLID: _PID}) + df[_PID] = coerce_patient_id(df[_PID]) + for col in ("dtstart", "dtend"): + if col in df.columns: + df[col] = coerce_date(df[col]) + df = deduplicate(df) + return df diff --git a/src/ehrdata/io/source/extract.py b/src/ehrdata/io/source/extract.py new file mode 100644 index 00000000..a713f0cd --- /dev/null +++ b/src/ehrdata/io/source/extract.py @@ -0,0 +1,182 @@ +"""Generic extraction helpers for the non-OMOP source ingestion layer. + +Provides DataFrame-level utilities that correspond to the repeating patterns +found across the original MarketScan, LCED, and CPRD R scripts: + +- ``union_tables`` mirrors dplyr ``union_all() + distinct_all()`` +- ``unnest_codes`` mirrors SQL ``unnest(array[dx1, dx2, ...])`` +- ``read_zipped_tsv`` / ``read_zipped_tsvs`` mirror + ``fread(cmd=paste("unzip -p", file))`` +- ``read_csv_with_duckdb`` mirrors ``data.table::fread`` for larger files +""" + +from __future__ import annotations + +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING + +import pandas as pd + +if TYPE_CHECKING: + pass + + +def union_tables(dfs: list[pd.DataFrame]) -> pd.DataFrame: + """Concatenate DataFrames and remove duplicate rows. + + Mirrors the ``union_all() |> distinct_all()`` idiom used in the LCED and + MarketScan R ETL scripts when assembling diagnosis/procedure tables from + multiple source views. + + Args: + dfs: Non-empty list of DataFrames with compatible columns. + + Returns: + Single deduplicated DataFrame; index reset to 0-based integers. + + Raises: + ValueError: If *dfs* is empty. + """ + if not dfs: + raise ValueError("dfs must contain at least one DataFrame") + return pd.concat(dfs, ignore_index=True).drop_duplicates().reset_index(drop=True) + + +def unnest_codes( + df: pd.DataFrame, + id_cols: list[str], + code_cols: list[str], + *, + value_name: str = "code", +) -> pd.DataFrame: + """Pivot wide-format code columns to long format, dropping null values. + + Mirrors ``SELECT unnest(array[dx1, dx2, dx3]) AS dx FROM ...`` used in + the LCED and MarketScan ETL to expand multi-code rows into one row per + code. + + Example:: + + # Wide: (patient_id=1, eventdate=…, dx1="E11", dx2="I10", dx3=None) + # Long: (patient_id=1, eventdate=…, dx="E11") + # (patient_id=1, eventdate=…, dx="I10") + + Args: + df: Wide-format DataFrame. + id_cols: Columns to keep as identifiers (e.g. ``["patient_id", "eventdate"]``). + code_cols: Wide code columns to unnest (e.g. ``["dx1", "dx2", "dx3"]``). + value_name: Name for the resulting long-format code column. + + Returns: + Long-format DataFrame with columns ``id_cols + [value_name]``, null + codes removed and duplicate rows dropped; index reset. + """ + long = df[id_cols + code_cols].melt( + id_vars=id_cols, + value_vars=code_cols, + value_name=value_name, + ) + return ( + long.dropna(subset=[value_name]) + .drop(columns="variable") + .drop_duplicates() + .reset_index(drop=True) + ) + + +def read_zipped_tsv( + zip_path: Path | str, + member: str, + *, + usecols: list[str] | None = None, + **kwargs, +) -> pd.DataFrame: + """Read a single TSV member from a zip archive. + + Mirrors ``fread(cmd=paste("unzip -p", file), sep="\\t")`` used in the + CPRD R ETL to stream individual extract files without unzipping to disk. + + Args: + zip_path: Path to the ``.zip`` file. + member: Filename of the TSV inside the archive. + usecols: Columns to select; all columns if ``None``. + **kwargs: Forwarded to :func:`pandas.read_csv`. + + Returns: + DataFrame with the member's contents. + """ + with zipfile.ZipFile(zip_path) as zf: + with zf.open(member) as fh: + return pd.read_csv(fh, sep="\t", usecols=usecols, **kwargs) + + +def read_zipped_tsvs( + zip_path: Path | str, + *, + pattern: str | None = None, + usecols: list[str] | None = None, + **kwargs, +) -> pd.DataFrame: + """Read and concatenate all matching TSV members from a zip archive. + + Mirrors the ``foreach``-based parallel unzip-and-bind pattern in the + CPRD ETL that combined Controls and Main extract files. + + Args: + zip_path: Path to the ``.zip`` file. + pattern: Optional substring to filter member names by; all ``.txt``, + ``.tsv``, and ``.csv`` members are included if ``None``. + usecols: Columns to select from each member. + **kwargs: Forwarded to :func:`pandas.read_csv`. + + Returns: + Concatenated DataFrame from all matched members; duplicates preserved + (call :func:`union_tables` afterwards to deduplicate). + """ + _READABLE = (".txt", ".tsv", ".csv") + with zipfile.ZipFile(zip_path) as zf: + members = [m for m in zf.namelist() if m.endswith(_READABLE)] + if pattern is not None: + members = [m for m in members if pattern in m] + dfs = [] + for member in members: + with zf.open(member) as fh: + dfs.append(pd.read_csv(fh, sep="\t", usecols=usecols, **kwargs)) + if not dfs: + return pd.DataFrame(columns=usecols or []) + return pd.concat(dfs, ignore_index=True) + + +def read_csv_with_duckdb( + path: Path | str, + *, + columns: list[str] | None = None, + where: str | None = None, +) -> pd.DataFrame: + """Read a CSV file (or glob pattern) via DuckDB for efficient loading. + + Use this instead of :func:`pandas.read_csv` when the source file is large + or when a ``WHERE`` clause can substantially reduce the loaded rows. + + Args: + path: Path to a CSV file or a glob pattern (e.g. ``"data/*.csv"``). + DuckDB resolves globs and unions matching files automatically. + columns: Columns to project; all columns if ``None``. + where: Optional SQL ``WHERE`` clause applied inside DuckDB before + materialising results (e.g. ``"patient_id IN (1, 2, 3)"``). + + Returns: + DataFrame with the query results. + """ + import duckdb + + col_list = ", ".join(columns) if columns else "*" + sql = f"SELECT {col_list} FROM read_csv_auto('{path}')" + if where: + sql += f" WHERE {where}" + con = duckdb.connect(database=":memory:") + try: + return con.execute(sql).fetchdf() + finally: + con.close() \ No newline at end of file diff --git a/src/ehrdata/io/source/normalize.py b/src/ehrdata/io/source/normalize.py new file mode 100644 index 00000000..add3197c --- /dev/null +++ b/src/ehrdata/io/source/normalize.py @@ -0,0 +1,221 @@ +"""Normalization helpers for the non-OMOP source ingestion layer. + +Provides atomic transformations (ID coercion, date parsing, ICD version +inference, deduplication, sorting) and composite ``normalize_*`` pipelines +that apply them in the correct order for each canonical table type. + +All functions return new DataFrames and do not mutate their inputs. +""" + +from __future__ import annotations + +import pandas as pd + + +# ICD-9 uses E-codes (external causes) and V-codes (supplemental factors). +# When ``dxver`` is missing in a claims record, a leading E or V reliably +# identifies ICD-9 coding — heuristic taken from the original LCED ETL. +_ICD9_LEADING_CHARS = frozenset("EV") + + +def coerce_patient_id(series: pd.Series) -> pd.Series: + """Cast patient identifiers to strings and strip surrounding whitespace. + + Handles integer primary keys (``enrolid``, ``patid``) as well as string + keys with padding artefacts from CSV exports. + + Args: + series: Raw patient ID column of any dtype. + + Returns: + String series with whitespace stripped and original index preserved. + """ + return series.astype(str).str.strip() + + +def coerce_date(series: pd.Series, formats: list[str] | None = None) -> pd.Series: + """Parse a date series to ``datetime64[ns]``, trying multiple formats. + + Unparseable values become ``NaT`` rather than raising. + + Args: + series: Raw date column (strings, objects, or already datetime). + formats: Ordered list of ``strptime`` format strings to attempt before + falling back to pandas auto-inference. Useful when the source has + a known non-ISO format (e.g. ``"%d/%m/%Y"`` for CPRD extracts). + + Returns: + Series of ``datetime64[ns]`` with the original index preserved. + """ + if pd.api.types.is_datetime64_any_dtype(series): + return series + if formats: + for fmt in formats: + parsed = pd.to_datetime(series, format=fmt, errors="coerce") + if parsed.notna().any(): + return parsed + return pd.to_datetime(series, errors="coerce") + + +def infer_icd_version( + df: pd.DataFrame, + *, + dx_col: str = "dx", + dxver_col: str = "dxver", +) -> pd.DataFrame: + """Fill missing ICD version flags using code-prefix heuristics. + + When ``dxver`` is ``None`` / ``NaN`` and the code starts with ``E`` or + ``V`` the version is set to ``"9"`` (ICD-9). All other missing values are + left as ``None``. Existing non-null values are never overwritten. + + This reproduces the LCED ETL logic:: + + dxver = if_else(is.na(dxver) & substr(dx,1,1) %in% c("E","V"), "9", dxver) + + Args: + df: DataFrame containing diagnosis codes and a version column. + dx_col: Column holding the raw ICD code string. + dxver_col: Column holding the version flag + (``"0"`` = ICD-10, ``"9"`` = ICD-9). + + Returns: + Copy of *df* with ``dxver`` filled where inferrable. + """ + df = df.copy() + # Cast to object so string assignment into a float NaN column doesn't warn. + df[dxver_col] = df[dxver_col].astype(object) + missing_ver = df[dxver_col].isna() + starts_icd9 = df[dx_col].str[:1].isin(_ICD9_LEADING_CHARS) + df.loc[missing_ver & starts_icd9, dxver_col] = "9" + return df + + +def deduplicate(df: pd.DataFrame, *, subset: list[str] | None = None) -> pd.DataFrame: + """Remove duplicate rows, optionally restricted to a column subset. + + Args: + df: Input DataFrame. + subset: Columns to consider for duplication detection; all columns if + ``None``. + + Returns: + DataFrame with duplicates removed and index reset to 0-based integers. + """ + return df.drop_duplicates(subset=subset).reset_index(drop=True) + + +def sort_events( + df: pd.DataFrame, + *, + patient_col: str = "patient_id", + date_col: str = "eventdate", +) -> pd.DataFrame: + """Sort a clinical event table by patient then event date. + + Rows with missing dates are placed last, matching the ``arrange()`` + behaviour in the original R ETL scripts. + + Args: + df: Clinical event DataFrame containing patient and date columns. + patient_col: Name of the patient identifier column. + date_col: Name of the event date column. + + Returns: + Sorted DataFrame with index reset. + """ + return df.sort_values([patient_col, date_col], na_position="last").reset_index(drop=True) + + +# --------------------------------------------------------------------------- +# Composite normalisation pipelines +# --------------------------------------------------------------------------- + + +def normalize_diagnosis(df: pd.DataFrame) -> pd.DataFrame: + """Normalise a raw diagnosis DataFrame to the canonical schema. + + Applies (in order): patient ID coercion, date parsing, ICD version + inference, deduplication, and chronological sort. + + Args: + df: Raw diagnosis DataFrame. Must contain ``patient_id``, + ``eventdate``, and ``dx``. ``dxver`` is optional; if absent it is + added as a column of ``None`` values before inference. + + Returns: + Normalised diagnosis DataFrame conforming to + :data:`~ehrdata.io.source.schema.DIAGNOSIS`. + """ + df = df.copy() + df["patient_id"] = coerce_patient_id(df["patient_id"]) + df["eventdate"] = coerce_date(df["eventdate"]) + if "dxver" not in df.columns: + df["dxver"] = None + df = infer_icd_version(df) + df = deduplicate(df) + df = sort_events(df) + return df + + +def normalize_therapy(df: pd.DataFrame) -> pd.DataFrame: + """Normalise a raw therapy DataFrame to the canonical schema. + + Coerces patient IDs and all date columns, then deduplicates. + + Args: + df: Raw therapy DataFrame. Must contain ``patient_id``. Date columns + ``prescription_date``, ``start_date``, ``fill_date``, and + ``end_date`` are parsed when present. + + Returns: + Normalised therapy DataFrame conforming to + :data:`~ehrdata.io.source.schema.THERAPY`. + """ + df = df.copy() + df["patient_id"] = coerce_patient_id(df["patient_id"]) + for col in ("prescription_date", "start_date", "fill_date", "end_date"): + if col in df.columns: + df[col] = coerce_date(df[col]) + else: + df[col] = pd.NaT + df = deduplicate(df) + return df + + +def normalize_labtest(df: pd.DataFrame) -> pd.DataFrame: + """Normalise a raw lab test DataFrame to the canonical schema. + + Args: + df: Raw labtest DataFrame. Must contain ``patient_id`` and + ``eventdate``. + + Returns: + Normalised labtest DataFrame conforming to + :data:`~ehrdata.io.source.schema.LABTEST`. + """ + df = df.copy() + df["patient_id"] = coerce_patient_id(df["patient_id"]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = deduplicate(df) + df = sort_events(df) + return df + + +def normalize_procedure(df: pd.DataFrame) -> pd.DataFrame: + """Normalise a raw procedure DataFrame to the canonical schema. + + Args: + df: Raw procedure DataFrame. Must contain ``patient_id``, + ``eventdate``, and ``proc``. + + Returns: + Normalised procedure DataFrame conforming to + :data:`~ehrdata.io.source.schema.PROCEDURE`. + """ + df = df.copy() + df["patient_id"] = coerce_patient_id(df["patient_id"]) + df["eventdate"] = coerce_date(df["eventdate"]) + df = deduplicate(df) + df = sort_events(df) + return df diff --git a/src/ehrdata/io/source/schema.py b/src/ehrdata/io/source/schema.py new file mode 100644 index 00000000..d923a188 --- /dev/null +++ b/src/ehrdata/io/source/schema.py @@ -0,0 +1,174 @@ +"""Canonical table schemas for the non-OMOP source ingestion layer. + +Each :class:`TableSchema` instance describes one output table produced by a +source adapter (MarketScan, LCED, CPRD). They are the single source of truth +for column names, expected dtypes, and nullability across all adapters and +tests. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pandas as pd + + +@dataclass(frozen=True) +class ColumnSpec: + """Specification for a single column in a canonical table. + + Args: + name: Column name. + dtype: Pandas dtype string (e.g. ``"object"``, ``"datetime64[ns]"``). + nullable: Whether the column may contain ``NaN`` / ``NaT`` values. + """ + + name: str + dtype: str + nullable: bool = True + + +@dataclass(frozen=True) +class TableSchema: + """Canonical schema for one output table. + + Args: + name: Table name (used as key in :data:`ALL_SCHEMAS`). + columns: Ordered column specifications. + """ + + name: str + columns: tuple[ColumnSpec, ...] + + @property + def column_names(self) -> list[str]: + """Ordered list of column names.""" + return [c.name for c in self.columns] + + def empty(self) -> pd.DataFrame: + """Return an empty :class:`~pandas.DataFrame` typed to this schema. + + Returns: + Zero-row DataFrame with columns and dtypes matching the schema. + """ + return pd.DataFrame({c.name: pd.Series(dtype=c.dtype) for c in self.columns}) + + def validate(self, df: pd.DataFrame, *, strict: bool = False) -> list[str]: + """Return validation error messages for *df* against this schema. + + Args: + df: DataFrame to validate. + strict: If ``True``, also report columns present in *df* but not in + the schema. + + Returns: + List of error strings; empty list means the DataFrame is valid. + """ + errors: list[str] = [] + schema_cols = {c.name for c in self.columns} + for col in self.columns: + if col.name not in df.columns: + errors.append(f"Missing required column '{col.name}'") + if strict: + extra = sorted(set(df.columns) - schema_cols) + if extra: + errors.append(f"Unexpected columns: {extra}") + return errors + + +# --------------------------------------------------------------------------- +# Canonical table definitions +# --------------------------------------------------------------------------- + +DIAGNOSIS = TableSchema( + name="diagnosis", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("dxver", "object"), # "0" = ICD-10, "9" = ICD-9, None = unknown + ColumnSpec("eventdate", "datetime64[ns]"), + ColumnSpec("dx", "object", nullable=False), + ), +) + +THERAPY = TableSchema( + name="therapy", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("prescription_date", "datetime64[ns]"), + ColumnSpec("start_date", "datetime64[ns]"), + ColumnSpec("fill_date", "datetime64[ns]"), + ColumnSpec("end_date", "datetime64[ns]"), + ColumnSpec("refill", "Int64"), + ColumnSpec("rxcui", "object"), + ColumnSpec("ndc11", "object"), + ColumnSpec("ingredient", "object"), + ), +) + +LABTEST = TableSchema( + name="labtest", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("eventdate", "datetime64[ns]"), + ColumnSpec("value", "object"), + ColumnSpec("valuecat", "object"), + ColumnSpec("unit", "object"), + ColumnSpec("loinc", "object"), + ), +) + +PROCEDURE = TableSchema( + name="procedure", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("proctype", "object"), + ColumnSpec("eventdate", "datetime64[ns]"), + ColumnSpec("proc", "object", nullable=False), + ), +) + +PATINFO = TableSchema( + name="patinfo", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("dobyr", "Int64"), + ColumnSpec("sex", "object"), + ), +) + +INSURANCE = TableSchema( + name="insurance", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("svcdate", "datetime64[ns]"), + ColumnSpec("cob", "float64"), # coordination of benefits amount + ColumnSpec("coins", "float64"), # coinsurance amount + ColumnSpec("copay", "float64"), + ), +) + +PROVIDER = TableSchema( + name="provider", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("dtstart", "datetime64[ns]"), + ColumnSpec("dtend", "datetime64[ns]"), + ColumnSpec("plantyp", "object"), + ColumnSpec("rx", "object"), # pharmacy coverage flag + ColumnSpec("hlthplan", "object"), + ), +) + +HABIT = TableSchema( + name="habit", + columns=( + ColumnSpec("patient_id", "object", nullable=False), + ColumnSpec("encounter_date", "datetime64[ns]"), + ColumnSpec("mapped_question_answer", "object"), + ), +) + +ALL_SCHEMAS: dict[str, TableSchema] = { + s.name: s + for s in (DIAGNOSIS, THERAPY, LABTEST, PROCEDURE, PATINFO, INSURANCE, PROVIDER, HABIT) +} \ No newline at end of file diff --git a/src/ehrdata/io/source/to_ehrdata.py b/src/ehrdata/io/source/to_ehrdata.py new file mode 100644 index 00000000..792add2f --- /dev/null +++ b/src/ehrdata/io/source/to_ehrdata.py @@ -0,0 +1,183 @@ +"""Bridge from canonical source tables to :class:`~ehrdata.EHRData`. + +Converts the flat event DataFrames produced by the source adapters +(MarketScan, LCED, CPRD) into a binary presence matrix in an +:class:`~ehrdata.EHRData` object. + +Mapping +------- +- **obs** — one row per patient, taken from the canonical PATINFO table + (``patient_id`` becomes the index; ``dobyr`` and ``sex`` are kept as + annotation columns). +- **var** — one row per unique clinical concept encountered across all + supplied event tables. The index is formatted as ``"{source}:{code}"`` + (e.g. ``"diagnosis:E11.9"``, ``"therapy:metformin"``, ``"labtest:14749-6"``) + so that codes from different coding systems never collide. Columns + ``concept_source`` and ``concept_code`` unpack the index for easy querying. +- **X** — ``float64`` binary presence matrix of shape ``(n_obs × n_var)``: + ``1.0`` when a patient has at least one event for that concept, ``0.0`` + otherwise. Patients absent from *patinfo* are silently excluded. + Duplicate events do not inflate the value beyond ``1.0``. +- **uns** — provenance metadata keyed ``source_io_source`` (the *source* + argument) and ``source_io_tables`` (list of table names that were provided). + +This function does **not** build a time dimension (R tensor). Users who need +interval-level temporal encoding should aggregate the canonical DataFrames +into time bins before calling this function, or use the OMOP +``setup_variables`` path for OMOP-formatted data. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd + +if TYPE_CHECKING: + from ehrdata import EHRData + + +def to_ehrdata( + patinfo: pd.DataFrame, + *, + diagnosis: pd.DataFrame | None = None, + therapy: pd.DataFrame | None = None, + labtest: pd.DataFrame | None = None, + procedure: pd.DataFrame | None = None, + source: str | None = None, +) -> "EHRData": + """Convert canonical source tables into an :class:`~ehrdata.EHRData` presence matrix. + + Args: + patinfo: Canonical PATINFO DataFrame (must include ``patient_id``, + ``dobyr``, ``sex``). Defines the observation set — only patients + present here appear as rows in the output. + diagnosis: Optional canonical DIAGNOSIS DataFrame. The ``dx`` column + supplies concept codes prefixed with ``"diagnosis:"``. + therapy: Optional canonical THERAPY DataFrame. The ``ingredient`` + column supplies concept codes prefixed with ``"therapy:"``. Rows + with null ``ingredient`` are skipped. + labtest: Optional canonical LABTEST DataFrame. The ``loinc`` column + supplies concept codes prefixed with ``"labtest:"``. Rows with + null ``loinc`` are skipped. + procedure: Optional canonical PROCEDURE DataFrame. The ``proc`` + column supplies concept codes prefixed with ``"procedure:"``. + source: Optional free-text label for the data source (e.g. + ``"marketscan"``, ``"lced"``, ``"cprd"``). Stored in + ``uns["source_io_source"]``. + + Returns: + :class:`~ehrdata.EHRData` with ``obs``, ``var``, ``X``, and ``uns`` + populated. When no event tables are provided, ``X`` has zero columns + and ``var`` is empty. + + Examples: + >>> import pandas as pd + >>> import ehrdata as ed + >>> patinfo = pd.DataFrame({"patient_id": ["P1", "P2"], "dobyr": [1960, 1975], "sex": ["M", "F"]}) + >>> diagnosis = pd.DataFrame({"patient_id": ["P1", "P1", "P2"], "dx": ["E11.9", "I10", "E11.9"], "dxver": [None, None, None], "eventdate": pd.NaT}) + >>> edata = ed.io.source.to_ehrdata(patinfo, diagnosis=diagnosis, source="example") + >>> edata.obs_names.tolist() + ['P1', 'P2'] + >>> "diagnosis:E11.9" in edata.var_names + True + """ + from ehrdata import EHRData + + # ---- obs: one row per patient ---------------------------------------- + obs = patinfo[["patient_id", "dobyr", "sex"]].copy() + obs = obs.drop_duplicates(subset=["patient_id"]) + obs = obs.set_index("patient_id") + obs.index = obs.index.astype(str) + obs.index.name = "patient_id" + patients: list[str] = obs.index.tolist() + patient_set: set[str] = set(patients) + patient_idx: dict[str, int] = {p: i for i, p in enumerate(patients)} + + # ---- collect (patient_id, concept) presence pairs -------------------- + concept_frames: list[pd.DataFrame] = [] + tables_used: list[str] = [] + + if diagnosis is not None: + _append_pairs(concept_frames, diagnosis, "patient_id", "dx", "diagnosis") + tables_used.append("diagnosis") + + if therapy is not None: + _append_pairs(concept_frames, therapy, "patient_id", "ingredient", "therapy") + tables_used.append("therapy") + + if labtest is not None: + _append_pairs(concept_frames, labtest, "patient_id", "loinc", "labtest") + tables_used.append("labtest") + + if procedure is not None: + _append_pairs(concept_frames, procedure, "patient_id", "proc", "procedure") + tables_used.append("procedure") + + uns: dict = { + "source_io_source": source, + "source_io_tables": tables_used, + } + + if not concept_frames: + var = pd.DataFrame( + {"concept_source": pd.Series(dtype=object), "concept_code": pd.Series(dtype=object)}, + ) + var.index.name = "concept" + X = np.zeros((len(patients), 0), dtype=np.float64) + return EHRData(X=X, obs=obs, var=var, uns=uns) + + # ---- deduplicate and restrict to known patients ----------------------- + all_pairs = pd.concat(concept_frames, ignore_index=True).drop_duplicates() + all_pairs["patient_id"] = all_pairs["patient_id"].astype(str) + all_pairs = all_pairs[all_pairs["patient_id"].isin(patient_set)] + + # ---- var: one row per unique concept ---------------------------------- + concepts: list[str] = sorted(all_pairs["concept"].unique()) + concept_idx: dict[str, int] = {c: i for i, c in enumerate(concepts)} + + var = pd.DataFrame( + { + "concept_source": pd.array([c.split(":", 1)[0] for c in concepts], dtype=object), + "concept_code": pd.array([c.split(":", 1)[1] for c in concepts], dtype=object), + }, + index=pd.Index(concepts, name="concept"), + ) + + # ---- X: binary presence matrix (obs × var) --------------------------- + pid_indices = all_pairs["patient_id"].map(patient_idx) + con_indices = all_pairs["concept"].map(concept_idx) + + valid = pid_indices.notna() & con_indices.notna() + rows = pid_indices[valid].astype(int).values + cols = con_indices[valid].astype(int).values + + X = np.zeros((len(patients), len(concepts)), dtype=np.float64) + X[rows, cols] = 1.0 + + return EHRData(X=X, obs=obs, var=var, uns=uns) + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _append_pairs( + frames: list[pd.DataFrame], + df: pd.DataFrame, + pid_col: str, + code_col: str, + prefix: str, +) -> None: + """Extract (patient_id, prefixed concept) pairs from *df* and append to *frames*.""" + sub = df[[pid_col, code_col]].dropna(subset=[code_col]) + if sub.empty: + return + frames.append( + pd.DataFrame({ + "patient_id": sub[pid_col].astype(str).values, + "concept": prefix + ":" + sub[code_col].astype(str).values, + }) + ) diff --git a/src/ehrdata/io/source/vocab/__init__.py b/src/ehrdata/io/source/vocab/__init__.py new file mode 100644 index 00000000..53e214a3 --- /dev/null +++ b/src/ehrdata/io/source/vocab/__init__.py @@ -0,0 +1,9 @@ +"""Vocabulary / reference-data loaders for the source ingestion layer. + +Each sub-module loads one external coding system and exposes a helper that +left-joins a DataFrame with the loaded mapping. +""" + +from . import icd, loinc, ndc, prodcode, readcode, rxnorm + +__all__ = ["icd", "loinc", "ndc", "prodcode", "readcode", "rxnorm"] diff --git a/src/ehrdata/io/source/vocab/icd.py b/src/ehrdata/io/source/vocab/icd.py new file mode 100644 index 00000000..090dd423 --- /dev/null +++ b/src/ehrdata/io/source/vocab/icd.py @@ -0,0 +1,86 @@ +"""ICD coding system utilities. + +Provides the E/V prefix heuristic used to infer ICD-9 version flags from raw +claims codes, plus stubs for GEM (General Equivalence Mapping) and CCS +(Clinical Classifications Software) look-ups that require external files. +""" + +from __future__ import annotations + +import pandas as pd + +# ICD-9 code series that begin with E (external causes) or V (supplemental +# factors) are unambiguously ICD-9 when the version flag is absent in claims +# data. Used by the LCED ETL and replicated in normalize.infer_icd_version. +ICD9_PREFIXES: frozenset[str] = frozenset("EV") + + +def classify_icd_version(series: pd.Series) -> pd.Series: + """Infer ICD version for each code based on first-character heuristics. + + Returns ``"9"`` for codes starting with ``E`` or ``V`` (ICD-9 E/V codes), + ``None`` for all other codes. This is a heuristic — codes that start with + ``E`` or ``V`` exist in ICD-10 as well, so the result is reliable only + when applied to codes whose version is already unknown and the context + suggests they originate from legacy claims data. + + Args: + series: Series of ICD code strings. + + Returns: + Series of ``"9"`` or ``None`` with the same index. + """ + return series.str[:1].map(lambda c: "9" if c in ICD9_PREFIXES else None) + + +def normalize_icd_code(code: str) -> str: + """Strip whitespace and upper-case an ICD code string. + + Args: + code: Raw ICD code string. + + Returns: + Cleaned code. + """ + return code.strip().upper() + + +# --------------------------------------------------------------------------- +# Stubs — require external files not vendored in this package +# --------------------------------------------------------------------------- + + +def load_gem_map(path: str) -> pd.DataFrame: # pragma: no cover + """Load an ICD-9 / ICD-10 General Equivalence Mapping (GEM) file. + + GEM files are published by CMS and map ICD-9-CM ↔ ICD-10-CM. They are + not vendored in this package due to their size; download from + https://www.cms.gov/medicare/coding-billing/icd-10-codes/icd-10-cm-and-gems. + + Args: + path: Path to a GEM flat file (pipe-delimited, as released by CMS). + + Returns: + DataFrame with columns ``source_code``, ``target_code``, and + ``flags``. + + Raises: + NotImplementedError: Always — implementation pending external data. + """ + raise NotImplementedError("GEM loader requires external CMS GEM file; not yet implemented.") + + +def load_ccs_map(path: str) -> pd.DataFrame: # pragma: no cover + """Load an AHRQ Clinical Classifications Software (CCS) mapping file. + + Args: + path: Path to the CCS CSV file (AHRQ format). + + Returns: + DataFrame with columns ``icd_code``, ``ccs_category``, and + ``ccs_description``. + + Raises: + NotImplementedError: Always — implementation pending external data. + """ + raise NotImplementedError("CCS loader requires external AHRQ CCS file; not yet implemented.") diff --git a/src/ehrdata/io/source/vocab/loinc.py b/src/ehrdata/io/source/vocab/loinc.py new file mode 100644 index 00000000..abe628ff --- /dev/null +++ b/src/ehrdata/io/source/vocab/loinc.py @@ -0,0 +1,67 @@ +"""LOINC (Logical Observation Identifiers Names and Codes) vocabulary loader. + +Loads a minimal LOINC reference table and provides a helper to join +human-readable component names onto a lab-test DataFrame. + +The full LOINC corpus (~750 MB) is not vendored in this package. Only a +small fixture-based subset is used in tests. Download the full table from +https://loinc.org/downloads/ and pass its path to :func:`load_loinc_map`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + + +def load_loinc_map(path: Path | str) -> pd.DataFrame: + """Load a LOINC reference CSV into a lookup DataFrame. + + The file must be comma-separated and contain at minimum the columns + ``loinc`` (or ``loinc_num``) and ``component``. Long-common-name and + other columns are preserved if present. + + Args: + path: Path to a LOINC CSV file. + + Returns: + DataFrame with ``loinc`` as the first column, followed by any + additional columns present in the file. + """ + df = pd.read_csv(path, dtype=str) + df.columns = df.columns.str.lower().str.strip() + # LOINC distributes the code field as "loinc_num" in the full table; + # normalise to "loinc" for consistency with our canonical schema. + if "loinc_num" in df.columns and "loinc" not in df.columns: + df = df.rename(columns={"loinc_num": "loinc"}) + df["loinc"] = df["loinc"].str.strip() + df["component"] = df["component"].str.strip() + return df + + +def join_component_by_loinc( + df: pd.DataFrame, + loinc_map: pd.DataFrame, + *, + loinc_col: str = "loinc", +) -> pd.DataFrame: + """Add a ``component`` column to *df* via a left join on LOINC code. + + Args: + df: DataFrame containing a LOINC code column. + loinc_map: Lookup DataFrame as returned by :func:`load_loinc_map`. + loinc_col: Name of the LOINC column in *df*. + + Returns: + Copy of *df* with a ``component`` column appended or replaced. + """ + df = df.copy() + if "component" in df.columns: + df = df.drop(columns="component") + lookup = loinc_map[["loinc", "component"]].drop_duplicates(subset="loinc") + return df.merge( + lookup.rename(columns={"loinc": loinc_col}), + on=loinc_col, + how="left", + ) diff --git a/src/ehrdata/io/source/vocab/ndc.py b/src/ehrdata/io/source/vocab/ndc.py new file mode 100644 index 00000000..396820c6 --- /dev/null +++ b/src/ehrdata/io/source/vocab/ndc.py @@ -0,0 +1,68 @@ +"""NDC-to-ingredient vocabulary loader. + +The mapping file (``ndc ingredient map.txt``) shipped with the IBM LCED +Coding System has 52,859 rows with columns ``ndc11``, ``rxcui``, and +``ingredient``. This module loads that file and provides a helper to join +ingredient information onto a therapy DataFrame by NDC-11 code. + +NDC-11 codes are zero-padded to exactly 11 characters on load so that +matching is robust to leading-zero loss during CSV export. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +_NDC11_LEN = 11 + + +def load_ndc_ingredient_map(path: Path | str) -> pd.DataFrame: + """Load the NDC-to-ingredient mapping file. + + Args: + path: Path to ``ndc ingredient map.txt`` (comma-separated, with + header ``ndc11,rxcui,ingredient``). + + Returns: + DataFrame with columns ``ndc11`` (zero-padded 11-char string), + ``rxcui`` (string), and ``ingredient`` (string). + """ + df = pd.read_csv(path, dtype=str) + df.columns = df.columns.str.lower().str.strip() + df["ndc11"] = df["ndc11"].str.strip().str.zfill(_NDC11_LEN) + df["rxcui"] = df["rxcui"].str.strip() + df["ingredient"] = df["ingredient"].str.strip() + return df[["ndc11", "rxcui", "ingredient"]] + + +def join_ingredient_by_ndc( + df: pd.DataFrame, + ndc_map: pd.DataFrame, + *, + ndc_col: str = "ndc11", +) -> pd.DataFrame: + """Add an ``ingredient`` column to *df* via a left join on NDC code. + + If *df* already contains an ``ingredient`` column it is overwritten with + the joined values (preserving ``NaN`` where no match is found). + + Args: + df: DataFrame containing an NDC column. + ndc_map: Mapping DataFrame as returned by :func:`load_ndc_ingredient_map`. + ndc_col: Name of the NDC column in *df*. + + Returns: + Copy of *df* with an ``ingredient`` column appended or replaced. + """ + df = df.copy() + if "ingredient" in df.columns: + df = df.drop(columns="ingredient") + ndc_lookup = ndc_map[["ndc11", "ingredient"]].drop_duplicates(subset="ndc11") + merged = df.merge( + ndc_lookup.rename(columns={"ndc11": ndc_col}), + on=ndc_col, + how="left", + ) + return merged diff --git a/src/ehrdata/io/source/vocab/prodcode.py b/src/ehrdata/io/source/vocab/prodcode.py new file mode 100644 index 00000000..3f45800f --- /dev/null +++ b/src/ehrdata/io/source/vocab/prodcode.py @@ -0,0 +1,64 @@ +"""CPRD product code vocabulary helpers. + +Loads the CPRD ``product.txt`` / ``product.csv`` lookup (prodcode → drug +substance) and provides a helper to left-join drug substance onto a DataFrame +by prodcode. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + + +def load_product_map(path: Path | str) -> pd.DataFrame: + """Load CPRD ``product.txt`` or ``product.csv`` → (prodcode, drugsubstance). + + Both the raw ``product.txt`` (tab-delimited) and the derived ``product.csv`` + (comma-delimited, may contain a ``drugsubstance.updated`` column) are + supported. When ``drugsubstance.updated`` is present it is preferred over + the raw ``drugsubstance`` column so that the normalised substance names + produced by the Preparation step are used. + + Args: + path: Path to ``product.txt`` or ``product.csv``. + + Returns: + DataFrame with columns ``prodcode`` (str) and ``drugsubstance`` (str), + deduplicated on ``prodcode``. + """ + sep = "," if str(path).endswith(".csv") else "\t" + df = pd.read_csv(path, sep=sep, dtype=str) + df.columns = [c.lower().strip() for c in df.columns] + ingredient_col = "drugsubstance.updated" if "drugsubstance.updated" in df.columns else "drugsubstance" + df = df[["prodcode", ingredient_col]].rename(columns={ingredient_col: "drugsubstance"}) + df["prodcode"] = df["prodcode"].str.strip() + df["drugsubstance"] = df["drugsubstance"].str.strip() + return df.drop_duplicates(subset=["prodcode"]).reset_index(drop=True) + + +def join_drugsubstance_by_prodcode( + df: pd.DataFrame, + product_map: pd.DataFrame, + *, + prodcode_col: str = "prodcode", +) -> pd.DataFrame: + """Left-join ``drugsubstance`` (ingredient) onto *df* via *prodcode_col*. + + Rows whose prodcode is absent from the map receive ``NaN`` for + ``drugsubstance``. *df* is not mutated. + + Args: + df: Source DataFrame containing a prodcode column. + product_map: Mapping DataFrame as returned by :func:`load_product_map`. + prodcode_col: Column in *df* that holds prodcode values. + + Returns: + Copy of *df* with a ``drugsubstance`` column appended; index reset. + """ + map_ = product_map[["prodcode", "drugsubstance"]].drop_duplicates(subset=["prodcode"]) + result = df.copy().merge(map_, left_on=prodcode_col, right_on="prodcode", how="left") + if prodcode_col != "prodcode" and "prodcode" in result.columns: + result = result.drop(columns=["prodcode"]) + return result.reset_index(drop=True) diff --git a/src/ehrdata/io/source/vocab/readcode.py b/src/ehrdata/io/source/vocab/readcode.py new file mode 100644 index 00000000..eac691d1 --- /dev/null +++ b/src/ehrdata/io/source/vocab/readcode.py @@ -0,0 +1,58 @@ +"""CPRD Read code vocabulary helpers. + +Loads the CPRD ``medical.txt`` lookup (medcode → Read code) and provides a +helper to left-join Read codes onto a DataFrame by medcode. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + + +def load_medical_map(path: Path | str) -> pd.DataFrame: + """Load CPRD ``medical.txt`` → (medcode, readcode) mapping. + + ``medical.txt`` is tab-delimited with at least the columns ``medcode``, + ``readcode``, and ``desc``. Column names are lower-cased and stripped + before processing; only ``medcode`` and ``readcode`` are returned. + + Args: + path: Path to ``medical.txt`` (or a compatible tab-delimited file). + + Returns: + DataFrame with columns ``medcode`` (str) and ``readcode`` (str), + deduplicated on ``medcode``. + """ + df = pd.read_csv(path, sep="\t", dtype=str) + df.columns = [c.lower().strip() for c in df.columns] + df["medcode"] = df["medcode"].str.strip() + df["readcode"] = df["readcode"].str.strip() + return df[["medcode", "readcode"]].drop_duplicates(subset=["medcode"]).reset_index(drop=True) + + +def join_readcode_by_medcode( + df: pd.DataFrame, + medical_map: pd.DataFrame, + *, + medcode_col: str = "medcode", +) -> pd.DataFrame: + """Left-join ``readcode`` onto *df* using *medcode_col* as the key. + + Rows in *df* whose medcode is absent from the map receive ``NaN`` for + ``readcode``. *df* is not mutated. + + Args: + df: Source DataFrame containing a medcode column. + medical_map: Mapping DataFrame as returned by :func:`load_medical_map`. + medcode_col: Column in *df* that holds medcode values. + + Returns: + Copy of *df* with a ``readcode`` column appended; index reset. + """ + map_ = medical_map[["medcode", "readcode"]].drop_duplicates(subset=["medcode"]) + result = df.copy().merge(map_, left_on=medcode_col, right_on="medcode", how="left") + if medcode_col != "medcode" and "medcode" in result.columns: + result = result.drop(columns=["medcode"]) + return result.reset_index(drop=True) diff --git a/src/ehrdata/io/source/vocab/rxnorm.py b/src/ehrdata/io/source/vocab/rxnorm.py new file mode 100644 index 00000000..f21b7a09 --- /dev/null +++ b/src/ehrdata/io/source/vocab/rxnorm.py @@ -0,0 +1,62 @@ +"""RxCUI-to-ingredient vocabulary loader. + +The mapping file (``rxcui ingredient map.txt``) shipped with the IBM LCED +Coding System has 24,659 rows with columns ``V1`` (row index), ``rxcui``, and +``ingredient``. This module loads that file and provides a helper to join +ingredient information onto a therapy DataFrame by RxNorm concept identifier. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + + +def load_rxcui_ingredient_map(path: Path | str) -> pd.DataFrame: + """Load the RxCUI-to-ingredient mapping file. + + Args: + path: Path to ``rxcui ingredient map.txt`` (comma-separated, with + header ``V1,rxcui,ingredient`` where ``V1`` is a row index). + + Returns: + DataFrame with columns ``rxcui`` (string) and ``ingredient`` (string). + """ + df = pd.read_csv(path, dtype=str) + df.columns = df.columns.str.lower().str.strip() + df["rxcui"] = df["rxcui"].str.strip() + df["ingredient"] = df["ingredient"].str.strip() + return df[["rxcui", "ingredient"]] + + +def join_ingredient_by_rxcui( + df: pd.DataFrame, + rxcui_map: pd.DataFrame, + *, + rxcui_col: str = "rxcui", +) -> pd.DataFrame: + """Add an ``ingredient`` column to *df* via a left join on RxCUI. + + If *df* already contains an ``ingredient`` column it is overwritten with + the joined values. + + Args: + df: DataFrame containing an RxCUI column. + rxcui_map: Mapping DataFrame as returned by + :func:`load_rxcui_ingredient_map`. + rxcui_col: Name of the RxCUI column in *df*. + + Returns: + Copy of *df* with an ``ingredient`` column appended or replaced. + """ + df = df.copy() + if "ingredient" in df.columns: + df = df.drop(columns="ingredient") + rxcui_lookup = rxcui_map[["rxcui", "ingredient"]].drop_duplicates(subset="rxcui") + merged = df.merge( + rxcui_lookup.rename(columns={"rxcui": rxcui_col}), + on=rxcui_col, + how="left", + ) + return merged diff --git a/tests/data/source_basic/diagnosis.csv b/tests/data/source_basic/diagnosis.csv new file mode 100644 index 00000000..9fb14767 --- /dev/null +++ b/tests/data/source_basic/diagnosis.csv @@ -0,0 +1,6 @@ +patient_id,dxver,eventdate,dx +1,0,2020-01-15,E11.9 +2,,2020-02-01,E10.9 +1,0,2020-01-15,E11.9 +3,,2020-03-10,V58.67 + 4 ,0,2020-04-01,I10 diff --git a/tests/data/source_basic/diagnosis_wide.csv b/tests/data/source_basic/diagnosis_wide.csv new file mode 100644 index 00000000..62db26fe --- /dev/null +++ b/tests/data/source_basic/diagnosis_wide.csv @@ -0,0 +1,4 @@ +patient_id,eventdate,dx1,dx2,dx3 +1,2020-01-15,E11.9,I10, +2,2020-02-01,E10.9,, +3,2020-03-10,V58.67,E11.9,I10 diff --git a/tests/data/source_basic/labtest.csv b/tests/data/source_basic/labtest.csv new file mode 100644 index 00000000..2ffee17f --- /dev/null +++ b/tests/data/source_basic/labtest.csv @@ -0,0 +1,5 @@ +patient_id,eventdate,value,valuecat,unit,loinc +1,2020-01-15,7.2,,mmol/L,14749-6 +2,2020-02-01,,high,mg/dL,2345-7 +1,2020-01-20,6.8,,mmol/L,14749-6 +2,2020-02-01,,high,mg/dL,2345-7 diff --git a/tests/data/source_basic/procedure.csv b/tests/data/source_basic/procedure.csv new file mode 100644 index 00000000..3c75e30d --- /dev/null +++ b/tests/data/source_basic/procedure.csv @@ -0,0 +1,5 @@ +patient_id,proctype,eventdate,proc +1,CPT,2020-01-15,99213 +2,CPT,2020-02-01,93000 +1,CPT,2020-01-15,99213 +3,,2020-03-10,99214 diff --git a/tests/data/source_basic/therapy.csv b/tests/data/source_basic/therapy.csv new file mode 100644 index 00000000..9e4e6be8 --- /dev/null +++ b/tests/data/source_basic/therapy.csv @@ -0,0 +1,4 @@ +patient_id,fill_date,end_date,refill,rxcui,ndc11,ingredient +1,2020-01-15,2020-02-14,0,1234567,12345678901,metformin +2,2020-02-01,,1,,98765432101,insulin glargine +1,2020-01-15,2020-02-14,0,1234567,12345678901,metformin diff --git a/tests/data/source_cprd/additional.tsv b/tests/data/source_cprd/additional.tsv new file mode 100644 index 00000000..7cd0ea20 --- /dev/null +++ b/tests/data/source_cprd/additional.tsv @@ -0,0 +1,4 @@ +patid enttype adid data1 data2 data3 data4 data5 data6 data7 +P001 4 A1 = 7.2 mmol/L normal 0 0 0 +P002 4 A3 > 9.1 mmol/L high 0 0 0 +P003 5 A4 = 52 mmol/mol normal 0 0 0 diff --git a/tests/data/source_cprd/clinical.tsv b/tests/data/source_cprd/clinical.tsv new file mode 100644 index 00000000..31aeb6a9 --- /dev/null +++ b/tests/data/source_cprd/clinical.tsv @@ -0,0 +1,6 @@ +patid eventdate medcode enttype adid constype consid +P001 15/03/2019 100 4 A1 1 C001 +P001 20/06/2019 200 5 A2 1 C002 +P002 10/01/2020 100 4 A3 1 C003 +P003 05/05/2018 300 7 A4 1 C004 +P001 15/03/2019 100 4 A1 1 C001 diff --git a/tests/data/source_cprd/patient.tsv b/tests/data/source_cprd/patient.tsv new file mode 100644 index 00000000..ab75cd30 --- /dev/null +++ b/tests/data/source_cprd/patient.tsv @@ -0,0 +1,4 @@ +patid dobyr sex pracid +P001 1955 Female 123 +P002 1962 Male 456 +P003 1978 Female 123 diff --git a/tests/data/source_cprd/practice.tsv b/tests/data/source_cprd/practice.tsv new file mode 100644 index 00000000..acb43ef5 --- /dev/null +++ b/tests/data/source_cprd/practice.tsv @@ -0,0 +1,3 @@ +pracid region lcd uts +123 London 31/12/2020 01/01/2000 +456 Yorkshire 31/12/2021 01/01/2001 diff --git a/tests/data/source_cprd/referral.tsv b/tests/data/source_cprd/referral.tsv new file mode 100644 index 00000000..2f21e536 --- /dev/null +++ b/tests/data/source_cprd/referral.tsv @@ -0,0 +1,3 @@ +patid eventdate medcode constype consid +P002 15/07/2020 400 3 C010 +P003 12/12/2019 200 3 C011 diff --git a/tests/data/source_cprd/test.tsv b/tests/data/source_cprd/test.tsv new file mode 100644 index 00000000..bb8af8a0 --- /dev/null +++ b/tests/data/source_cprd/test.tsv @@ -0,0 +1,4 @@ +patid eventdate medcode enttype data1 data2 data3 data4 data5 data6 data7 constype consid +P001 10/09/2019 500 7 = 6.5 % normal 1 C020 +P002 20/10/2020 500 7 = 140 mmol/L high 1 C021 +P003 01/03/2018 500 7 < 5.0 mmol/L normal 1 C022 diff --git a/tests/data/source_cprd/therapy.tsv b/tests/data/source_cprd/therapy.tsv new file mode 100644 index 00000000..ab95f210 --- /dev/null +++ b/tests/data/source_cprd/therapy.tsv @@ -0,0 +1,6 @@ +patid eventdate prodcode bnfcode qty issueseq +P001 01/04/2019 PROD1 02020100 28 1 +P001 01/07/2019 PROD1 02020100 28 2 +P002 15/05/2020 PROD2 03010000 56 1 +P003 10/02/2018 PROD3 04050000 30 1 +P001 01/04/2019 PROD1 02020100 28 1 diff --git a/tests/data/source_vocab/loinc_map.csv b/tests/data/source_vocab/loinc_map.csv new file mode 100644 index 00000000..ffc5bc31 --- /dev/null +++ b/tests/data/source_vocab/loinc_map.csv @@ -0,0 +1,11 @@ +loinc,component,long_common_name +14749-6,Glucose,Glucose [Moles/volume] in Serum or Plasma +2345-7,Glucose,Glucose [Mass/volume] in Serum or Plasma +4548-4,Hemoglobin A1c/Hemoglobin.total,Hemoglobin A1c/Hemoglobin.total in Blood +2823-3,Potassium,Potassium [Moles/volume] in Serum or Plasma +2951-2,Sodium,Sodium [Moles/volume] in Serum or Plasma +6768-6,Alkaline phosphatase,Alkaline phosphatase [Enzymatic activity/volume] in Serum or Plasma +1742-6,Alanine aminotransferase,Alanine aminotransferase [Enzymatic activity/volume] in Serum or Plasma +2160-0,Creatinine,Creatinine [Mass/volume] in Serum or Plasma +718-7,Hemoglobin,Hemoglobin [Mass/volume] in Blood +777-3,Platelets,Platelets [#/volume] in Blood diff --git a/tests/data/source_vocab/medical_map.txt b/tests/data/source_vocab/medical_map.txt new file mode 100644 index 00000000..4451c7fd --- /dev/null +++ b/tests/data/source_vocab/medical_map.txt @@ -0,0 +1,6 @@ +medcode readcode desc +100 A10..00 Ischaemic heart disease +200 C10..00 Diabetes mellitus +300 E12..00 Obesity +400 F45..00 Dementia +500 44J3.00 HbA1c measurement diff --git a/tests/data/source_vocab/ndc_ingredient_map.txt b/tests/data/source_vocab/ndc_ingredient_map.txt new file mode 100644 index 00000000..ce13b6ab --- /dev/null +++ b/tests/data/source_vocab/ndc_ingredient_map.txt @@ -0,0 +1,11 @@ +ndc11,rxcui,ingredient +57894015012,1100079,abiraterone +55111028130,311296,levofloxacin +00065063136,208601,dexamethasone/neomycin/polymyxin b +51079081920,197446,carisoprodol +00071015523,723,metformin +00310751030,4815,insulin glargine +00088221905,41493,pioglitazone +50458058530,321064,risperidone +00006074331,41126,sitagliptin +00169750612,274332,semaglutide diff --git a/tests/data/source_vocab/product_map.txt b/tests/data/source_vocab/product_map.txt new file mode 100644 index 00000000..41503ed5 --- /dev/null +++ b/tests/data/source_vocab/product_map.txt @@ -0,0 +1,4 @@ +prodcode drugsubstance strength formulation route bnfcode +PROD1 metformin 500mg tablet Oral 02020100 +PROD2 insulin glargine 100IU/mL injection Subcutaneous 03010000 +PROD3 atorvastatin 10mg tablet Oral 04050000 diff --git a/tests/data/source_vocab/rxcui_ingredient_map.txt b/tests/data/source_vocab/rxcui_ingredient_map.txt new file mode 100644 index 00000000..885deca3 --- /dev/null +++ b/tests/data/source_vocab/rxcui_ingredient_map.txt @@ -0,0 +1,11 @@ +V1,rxcui,ingredient +1,723,metformin +2,4815,insulin glargine +3,41493,pioglitazone +4,321064,risperidone +5,41126,sitagliptin +6,274332,semaglutide +7,1100079,abiraterone +8,311296,levofloxacin +9,197446,carisoprodol +10,208601,dexamethasone/neomycin/polymyxin b diff --git a/tests/io/test_source_cprd.py b/tests/io/test_source_cprd.py new file mode 100644 index 00000000..adc01c1c --- /dev/null +++ b/tests/io/test_source_cprd.py @@ -0,0 +1,474 @@ +"""Tests for Phase 4: CPRD adapter and CPRD-specific vocab loaders. + +Covers: +- vocab/readcode.py — load_medical_map, join_readcode_by_medcode +- vocab/prodcode.py — load_product_map, join_drugsubstance_by_prodcode +- adapters/cprd.py — build_diagnosis, build_therapy, build_labtest, + build_patinfo +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.adapters import cprd as cprd_adapter +from ehrdata.io.source.schema import DIAGNOSIS, LABTEST, PATINFO, THERAPY +from ehrdata.io.source.vocab import prodcode as prodcode_vocab +from ehrdata.io.source.vocab import readcode as readcode_vocab + +# --------------------------------------------------------------------------- +# Fixture paths +# --------------------------------------------------------------------------- + +_DATA = Path(__file__).parent.parent / "data" +_VOCAB = _DATA / "source_vocab" +_CPRD = _DATA / "source_cprd" + +MEDICAL_MAP_PATH = _VOCAB / "medical_map.txt" +PRODUCT_MAP_PATH = _VOCAB / "product_map.txt" + +CLINICAL_PATH = _CPRD / "clinical.tsv" +REFERRAL_PATH = _CPRD / "referral.tsv" +TEST_PATH = _CPRD / "test.tsv" +ADDITIONAL_PATH = _CPRD / "additional.tsv" +THERAPY_PATH = _CPRD / "therapy.tsv" +PATIENT_PATH = _CPRD / "patient.tsv" + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _read_tsv(path: Path, **kwargs) -> pd.DataFrame: + return pd.read_csv(path, sep="\t", dtype=str, **kwargs) + + +# --------------------------------------------------------------------------- +# TestLoadMedicalMap +# --------------------------------------------------------------------------- + + +class TestLoadMedicalMap: + def test_returns_dataframe(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + assert isinstance(result, pd.DataFrame) + + def test_columns(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + assert list(result.columns) == ["medcode", "readcode"] + + def test_row_count(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + assert len(result) == 5 + + def test_known_readcode(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + row = result[result["medcode"] == "100"] + assert row["readcode"].iloc[0] == "A10..00" + + def test_dtype_is_string(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + assert result["medcode"].dtype == object + assert result["readcode"].dtype == object + + def test_deduplicates_on_medcode(self): + # feeding a path where the same medcode appears twice + import io + data = "medcode\treadcode\tdesc\n100\tA10..00\tFoo\n100\tA10..00\tFoo duplicate\n" + import tempfile, os + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(data) + tmp = f.name + try: + result = readcode_vocab.load_medical_map(tmp) + assert result["medcode"].duplicated().sum() == 0 + finally: + os.unlink(tmp) + + def test_extra_columns_ignored(self): + result = readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + assert "desc" not in result.columns + + +# --------------------------------------------------------------------------- +# TestJoinReadcodeByMedcode +# --------------------------------------------------------------------------- + + +class TestJoinReadcodeByMedcode: + @pytest.fixture + def medical_map(self): + return readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + + def test_matched_row_gets_readcode(self, medical_map): + df = pd.DataFrame({"medcode": ["100"], "patient_id": ["P1"]}) + result = readcode_vocab.join_readcode_by_medcode(df, medical_map) + assert result["readcode"].iloc[0] == "A10..00" + + def test_unmatched_row_gets_nan(self, medical_map): + df = pd.DataFrame({"medcode": ["UNKNOWN"], "patient_id": ["P1"]}) + result = readcode_vocab.join_readcode_by_medcode(df, medical_map) + assert pd.isna(result["readcode"].iloc[0]) + + def test_row_count_preserved(self, medical_map): + df = pd.DataFrame({"medcode": ["100", "200", "UNKNOWN"]}) + result = readcode_vocab.join_readcode_by_medcode(df, medical_map) + assert len(result) == 3 + + def test_does_not_mutate_input(self, medical_map): + df = pd.DataFrame({"medcode": ["100"]}) + cols_before = list(df.columns) + readcode_vocab.join_readcode_by_medcode(df, medical_map) + assert list(df.columns) == cols_before + + def test_custom_medcode_column(self, medical_map): + df = pd.DataFrame({"mcode": ["200"]}) + result = readcode_vocab.join_readcode_by_medcode(df, medical_map, medcode_col="mcode") + assert result["readcode"].iloc[0] == "C10..00" + + +# --------------------------------------------------------------------------- +# TestLoadProductMap +# --------------------------------------------------------------------------- + + +class TestLoadProductMap: + def test_returns_dataframe(self): + result = prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + assert isinstance(result, pd.DataFrame) + + def test_columns(self): + result = prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + assert list(result.columns) == ["prodcode", "drugsubstance"] + + def test_row_count(self): + result = prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + assert len(result) == 3 + + def test_known_drugsubstance(self): + result = prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + row = result[result["prodcode"] == "PROD1"] + assert row["drugsubstance"].iloc[0] == "metformin" + + def test_prefers_updated_column(self): + import io, tempfile, os + data = "prodcode\tdrugsubstance\tdrugsubstance.updated\n" "PC1\traw name\tupdated name\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write(data) + tmp = f.name + try: + result = prodcode_vocab.load_product_map(tmp) + assert result["drugsubstance"].iloc[0] == "updated name" + finally: + os.unlink(tmp) + + def test_csv_separator(self): + import tempfile, os + data = "prodcode,drugsubstance\nPC1,aspirin\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: + f.write(data) + tmp = f.name + try: + result = prodcode_vocab.load_product_map(tmp) + assert result["drugsubstance"].iloc[0] == "aspirin" + finally: + os.unlink(tmp) + + +# --------------------------------------------------------------------------- +# TestJoinDrugsubstanceByProdcode +# --------------------------------------------------------------------------- + + +class TestJoinDrugsubstanceByProdcode: + @pytest.fixture + def product_map(self): + return prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + + def test_matched_row_gets_drugsubstance(self, product_map): + df = pd.DataFrame({"prodcode": ["PROD2"]}) + result = prodcode_vocab.join_drugsubstance_by_prodcode(df, product_map) + assert result["drugsubstance"].iloc[0] == "insulin glargine" + + def test_unmatched_row_gets_nan(self, product_map): + df = pd.DataFrame({"prodcode": ["UNKNOWN"]}) + result = prodcode_vocab.join_drugsubstance_by_prodcode(df, product_map) + assert pd.isna(result["drugsubstance"].iloc[0]) + + def test_row_count_preserved(self, product_map): + df = pd.DataFrame({"prodcode": ["PROD1", "PROD3", "UNKNOWN"]}) + result = prodcode_vocab.join_drugsubstance_by_prodcode(df, product_map) + assert len(result) == 3 + + def test_does_not_mutate_input(self, product_map): + df = pd.DataFrame({"prodcode": ["PROD1"]}) + cols_before = list(df.columns) + prodcode_vocab.join_drugsubstance_by_prodcode(df, product_map) + assert list(df.columns) == cols_before + + def test_custom_prodcode_column(self, product_map): + df = pd.DataFrame({"pcode": ["PROD3"]}) + result = prodcode_vocab.join_drugsubstance_by_prodcode(df, product_map, prodcode_col="pcode") + assert result["drugsubstance"].iloc[0] == "atorvastatin" + + +# --------------------------------------------------------------------------- +# Shared adapter fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def clinical(): + return _read_tsv(CLINICAL_PATH) + + +@pytest.fixture +def referral(): + return _read_tsv(REFERRAL_PATH) + + +@pytest.fixture +def test_data(): + return _read_tsv(TEST_PATH) + + +@pytest.fixture +def additional(): + return _read_tsv(ADDITIONAL_PATH) + + +@pytest.fixture +def therapy_data(): + return _read_tsv(THERAPY_PATH) + + +@pytest.fixture +def patient(): + return _read_tsv(PATIENT_PATH) + + +@pytest.fixture +def medical_map(): + return readcode_vocab.load_medical_map(MEDICAL_MAP_PATH) + + +@pytest.fixture +def product_map(): + return prodcode_vocab.load_product_map(PRODUCT_MAP_PATH) + + +# --------------------------------------------------------------------------- +# TestCprdBuildDiagnosis +# --------------------------------------------------------------------------- + + +class TestCprdBuildDiagnosis: + def test_schema_valid(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + errors = DIAGNOSIS.validate(result) + assert errors == [] + + def test_three_sources_contribute(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + # clinical: P001/P002/P003, referral: P002/P003, test: P001/P002/P003 + assert set(result["patient_id"]) >= {"P001", "P002", "P003"} + + def test_dxver_is_none_for_all_rows(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert result["dxver"].isna().all() + + def test_dxver_dtype_is_object(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert result["dxver"].dtype == object + + def test_medical_map_translates_to_readcode(self, clinical, referral, test_data, medical_map): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data, medical_map=medical_map) + # medcode 100 → A10..00; medcode 200 → C10..00 + assert "A10..00" in result["dx"].values + assert "C10..00" in result["dx"].values + + def test_no_medical_map_uses_raw_medcode(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert "100" in result["dx"].values + + def test_no_duplicate_rows(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + pids = result["patient_id"].tolist() + dates = result["eventdate"].tolist() + for i in range(len(pids) - 1): + if pids[i] == pids[i + 1]: + assert dates[i] <= dates[i + 1] or pd.isna(dates[i + 1]) + + def test_eventdate_is_datetime(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_cprd_date_format_parsed(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + # 15/03/2019 should parse to 2019-03-15 + p1_dates = result[result["patient_id"] == "P001"]["eventdate"] + assert (p1_dates == pd.Timestamp("2019-03-15")).any() + + def test_patient_id_is_string(self, clinical, referral, test_data): + result = cprd_adapter.build_diagnosis(clinical, referral, test_data) + assert result["patient_id"].dtype == object + + +# --------------------------------------------------------------------------- +# TestCprdBuildTherapy +# --------------------------------------------------------------------------- + + +class TestCprdBuildTherapy: + def test_schema_valid(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + errors = THERAPY.validate(result) + assert errors == [] + + def test_fill_date_is_eventdate(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + assert pd.api.types.is_datetime64_any_dtype(result["fill_date"]) + # 01/04/2019 → 2019-04-01 + assert (result["fill_date"] == pd.Timestamp("2019-04-01")).any() + + def test_product_map_provides_ingredient(self, therapy_data, product_map): + result = cprd_adapter.build_therapy(therapy_data, product_map=product_map) + assert "metformin" in result["ingredient"].values + + def test_no_product_map_gives_null_ingredient(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + assert result["ingredient"].isna().all() + + def test_prescription_start_end_dates_are_nat(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + for col in ("prescription_date", "start_date", "end_date"): + assert result[col].isna().all() + + def test_rxcui_and_ndc11_are_null(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + assert result["rxcui"].isna().all() + assert result["ndc11"].isna().all() + + def test_refill_is_nullable_int(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + assert result["refill"].dtype == "Int64" + + def test_no_duplicate_rows(self, therapy_data): + # clinical.tsv has a duplicate therapy row; should be removed + result = cprd_adapter.build_therapy(therapy_data) + assert not result.duplicated().any() + + def test_patient_id_is_string(self, therapy_data): + result = cprd_adapter.build_therapy(therapy_data) + assert result["patient_id"].dtype == object + + def test_three_unique_drugs_present(self, therapy_data, product_map): + result = cprd_adapter.build_therapy(therapy_data, product_map=product_map) + assert set(result["ingredient"].dropna()) >= {"metformin", "insulin glargine", "atorvastatin"} + + +# --------------------------------------------------------------------------- +# TestCprdBuildLabtest +# --------------------------------------------------------------------------- + + +class TestCprdBuildLabtest: + def test_schema_valid(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + errors = LABTEST.validate(result) + assert errors == [] + + def test_both_sources_contribute(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + # clinical+additional contributes P001/P002/P003 via adid join + # test_data also contributes P001/P002/P003 + assert set(result["patient_id"]) == {"P001", "P002", "P003"} + + def test_data2_mapped_to_value(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert "value" in result.columns + assert "7.2" in result["value"].values or 7.2 in result["value"].values + + def test_data3_mapped_to_unit(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert "mmol/L" in result["unit"].values + + def test_data4_mapped_to_valuecat(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert "normal" in result["valuecat"].values or "high" in result["valuecat"].values + + def test_loinc_is_null(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert result["loinc"].isna().all() + + def test_eventdate_is_datetime(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_entity_filter_reduces_rows(self, clinical, additional, test_data): + all_result = cprd_adapter.build_labtest(clinical, additional, test_data) + # only enttype 7 (from test_data only since additional+clinical join gives enttype 4/5) + filtered = cprd_adapter.build_labtest(clinical, additional, test_data, entity_enttypes={"7"}) + assert len(filtered) < len(all_result) + + def test_no_duplicate_rows(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, clinical, additional, test_data): + result = cprd_adapter.build_labtest(clinical, additional, test_data) + pids = result["patient_id"].tolist() + dates = result["eventdate"].tolist() + for i in range(len(pids) - 1): + if pids[i] == pids[i + 1]: + assert dates[i] <= dates[i + 1] or pd.isna(dates[i + 1]) + + +# --------------------------------------------------------------------------- +# TestCprdBuildPatinfo +# --------------------------------------------------------------------------- + + +class TestCprdBuildPatinfo: + def test_schema_valid(self, patient): + result = cprd_adapter.build_patinfo(patient) + errors = PATINFO.validate(result) + assert errors == [] + + def test_three_canonical_columns_only(self, patient): + result = cprd_adapter.build_patinfo(patient) + assert list(result.columns) == ["patient_id", "dobyr", "sex"] + + def test_all_patients_present(self, patient): + result = cprd_adapter.build_patinfo(patient) + assert set(result["patient_id"]) == {"P001", "P002", "P003"} + + def test_dobyr_is_nullable_int(self, patient): + result = cprd_adapter.build_patinfo(patient) + assert result["dobyr"].dtype == "Int64" + + def test_dobyr_values_correct(self, patient): + result = cprd_adapter.build_patinfo(patient) + p1 = result[result["patient_id"] == "P001"]["dobyr"].iloc[0] + assert p1 == 1955 + + def test_deduplicates_across_tables(self, patient): + result = cprd_adapter.build_patinfo(patient, patient.copy()) + assert not result.duplicated().any() + assert len(result) == 3 + + def test_patient_id_is_string(self, patient): + result = cprd_adapter.build_patinfo(patient) + assert result["patient_id"].dtype == object + + def test_pracid_column_not_in_output(self, patient): + result = cprd_adapter.build_patinfo(patient) + assert "pracid" not in result.columns diff --git a/tests/io/test_source_extract.py b/tests/io/test_source_extract.py new file mode 100644 index 00000000..225552f5 --- /dev/null +++ b/tests/io/test_source_extract.py @@ -0,0 +1,204 @@ +import io +import zipfile +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.extract import ( + read_csv_with_duckdb, + read_zipped_tsv, + read_zipped_tsvs, + union_tables, + unnest_codes, +) + +FIXTURE_DIR = Path("tests/data/source_basic") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_zip(members: dict[str, str]) -> bytes: + """Return in-memory zip bytes with the given {filename: tsv_content} map.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, content in members.items(): + zf.writestr(name, content) + return buf.getvalue() + + +# --------------------------------------------------------------------------- +# union_tables +# --------------------------------------------------------------------------- + + +class TestUnionTables: + def test_basic_concat(self): + df1 = pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}) + df2 = pd.DataFrame({"a": [3, 4], "b": ["z", "w"]}) + result = union_tables([df1, df2]) + assert len(result) == 4 + assert list(result.columns) == ["a", "b"] + + def test_removes_exact_duplicates(self): + df1 = pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}) + df2 = pd.DataFrame({"a": [2, 3], "b": ["y", "z"]}) + result = union_tables([df1, df2]) + assert len(result) == 3 + + def test_index_reset(self): + df1 = pd.DataFrame({"a": [10]}, index=[99]) + df2 = pd.DataFrame({"a": [20]}, index=[100]) + result = union_tables([df1, df2]) + assert list(result.index) == [0, 1] + + def test_single_dataframe(self): + df = pd.DataFrame({"a": [1, 1, 2]}) + result = union_tables([df]) + assert len(result) == 2 + + def test_empty_list_raises(self): + with pytest.raises(ValueError): + union_tables([]) + + def test_preserves_column_order(self): + df1 = pd.DataFrame({"x": [1], "y": [2], "z": [3]}) + result = union_tables([df1]) + assert list(result.columns) == ["x", "y", "z"] + + +# --------------------------------------------------------------------------- +# unnest_codes +# --------------------------------------------------------------------------- + + +class TestUnnestCodes: + def test_wide_to_long(self): + df = pd.read_csv(FIXTURE_DIR / "diagnosis_wide.csv") + result = unnest_codes(df, id_cols=["patient_id", "eventdate"], code_cols=["dx1", "dx2", "dx3"], value_name="dx") + assert "dx" in result.columns + assert "variable" not in result.columns + assert "dx1" not in result.columns + + def test_drops_null_codes(self): + df = pd.read_csv(FIXTURE_DIR / "diagnosis_wide.csv") + result = unnest_codes(df, id_cols=["patient_id", "eventdate"], code_cols=["dx1", "dx2", "dx3"], value_name="dx") + assert result["dx"].notna().all() + + def test_correct_row_count(self): + # Row1: dx1=E11.9, dx2=I10 → 2 codes + # Row2: dx1=E10.9 → 1 code + # Row3: dx1=V58.67, dx2=E11.9, dx3=I10 → 3 codes + # Total before dedup: 6; after dedup: 6 (all distinct patient×date×dx) + df = pd.read_csv(FIXTURE_DIR / "diagnosis_wide.csv") + result = unnest_codes(df, id_cols=["patient_id", "eventdate"], code_cols=["dx1", "dx2", "dx3"], value_name="dx") + assert len(result) == 6 + + def test_deduplicates_identical_long_rows(self): + df = pd.DataFrame({ + "pid": [1, 1], + "dx1": ["A01", "A01"], + "dx2": ["B02", "B02"], + }) + result = unnest_codes(df, id_cols=["pid"], code_cols=["dx1", "dx2"], value_name="dx") + assert len(result) == 2 # (pid=1,dx=A01) and (pid=1,dx=B02) + + def test_custom_value_name(self): + df = pd.DataFrame({"pid": [1], "p1": ["99213"], "p2": [None]}) + result = unnest_codes(df, id_cols=["pid"], code_cols=["p1", "p2"], value_name="proc") + assert "proc" in result.columns + + def test_all_null_returns_empty(self): + df = pd.DataFrame({"pid": [1, 2], "dx1": [None, None]}) + result = unnest_codes(df, id_cols=["pid"], code_cols=["dx1"]) + assert len(result) == 0 + + def test_index_reset(self): + df = pd.DataFrame({"pid": [1, 2], "dx1": ["A", "B"]}) + result = unnest_codes(df, id_cols=["pid"], code_cols=["dx1"]) + assert list(result.index) == [0, 1] + + +# --------------------------------------------------------------------------- +# read_zipped_tsv +# --------------------------------------------------------------------------- + + +class TestReadZippedTsv: + @pytest.fixture() + def zip_bytes(self, tmp_path): + content = "patient_id\teventdate\tdx\n1\t2020-01-15\tE11.9\n2\t2020-02-01\tI10\n" + zdata = _make_zip({"clinical.txt": content}) + p = tmp_path / "extract.zip" + p.write_bytes(zdata) + return p + + def test_reads_member(self, zip_bytes): + df = read_zipped_tsv(zip_bytes, "clinical.txt") + assert len(df) == 2 + assert list(df.columns) == ["patient_id", "eventdate", "dx"] + + def test_usecols_filter(self, zip_bytes): + df = read_zipped_tsv(zip_bytes, "clinical.txt", usecols=["patient_id", "dx"]) + assert list(df.columns) == ["patient_id", "dx"] + assert "eventdate" not in df.columns + + def test_accepts_string_path(self, zip_bytes): + df = read_zipped_tsv(str(zip_bytes), "clinical.txt") + assert len(df) == 2 + + +# --------------------------------------------------------------------------- +# read_zipped_tsvs +# --------------------------------------------------------------------------- + + +class TestReadZippedTsvs: + @pytest.fixture() + def zip_bytes(self, tmp_path): + tsv1 = "patient_id\tdx\n1\tE11.9\n2\tI10\n" + tsv2 = "patient_id\tdx\n3\tE10.9\n4\tJ45\n" + zdata = _make_zip({"controls/clinical.txt": tsv1, "main/clinical.txt": tsv2}) + p = tmp_path / "extract.zip" + p.write_bytes(zdata) + return p + + def test_unions_all_members(self, zip_bytes): + df = read_zipped_tsvs(zip_bytes) + assert len(df) == 4 + + def test_pattern_filter(self, zip_bytes): + df = read_zipped_tsvs(zip_bytes, pattern="controls") + assert len(df) == 2 + + def test_no_match_returns_empty(self, zip_bytes): + df = read_zipped_tsvs(zip_bytes, pattern="nonexistent") + assert len(df) == 0 + + def test_usecols(self, zip_bytes): + df = read_zipped_tsvs(zip_bytes, usecols=["patient_id"]) + assert list(df.columns) == ["patient_id"] + + +# --------------------------------------------------------------------------- +# read_csv_with_duckdb +# --------------------------------------------------------------------------- + + +class TestReadCsvWithDuckdb: + def test_reads_csv(self): + df = read_csv_with_duckdb(FIXTURE_DIR / "diagnosis.csv") + assert len(df) > 0 + assert "patient_id" in df.columns + + def test_column_projection(self): + df = read_csv_with_duckdb(FIXTURE_DIR / "diagnosis.csv", columns=["patient_id", "dx"]) + assert list(df.columns) == ["patient_id", "dx"] + assert "eventdate" not in df.columns + + def test_where_clause(self): + df = read_csv_with_duckdb(FIXTURE_DIR / "diagnosis.csv", where="dxver = '0'") + assert all(str(v) == "0" for v in df["dxver"]) diff --git a/tests/io/test_source_lced.py b/tests/io/test_source_lced.py new file mode 100644 index 00000000..8d0c8fe4 --- /dev/null +++ b/tests/io/test_source_lced.py @@ -0,0 +1,550 @@ +"""Tests for the IBM LCED adapter and Phase 3 vocab modules. + +Source DataFrames are constructed inline to mirror the minimum columns present +in the LCED PostgreSQL views. +""" + +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.adapters.lced import ( + build_diagnosis, + build_habit, + build_insurance, + build_labtest, + build_patinfo, + build_procedure, + build_provider, + build_therapy, +) +from ehrdata.io.source.schema import DIAGNOSIS, HABIT, INSURANCE, LABTEST, PATINFO, PROCEDURE, PROVIDER, THERAPY +from ehrdata.io.source.vocab.icd import ICD9_PREFIXES, classify_icd_version +from ehrdata.io.source.vocab.loinc import join_component_by_loinc, load_loinc_map +from ehrdata.io.source.vocab.ndc import load_ndc_ingredient_map +from ehrdata.io.source.vocab.rxnorm import load_rxcui_ingredient_map + +VOCAB_DIR = Path("tests/data/source_vocab") + + +# --------------------------------------------------------------------------- +# Vocab fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def ndc_map(): + return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") + + +@pytest.fixture() +def rxcui_map(): + return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt") + + +@pytest.fixture() +def loinc_map(): + return load_loinc_map(VOCAB_DIR / "loinc_map.csv") + + +# --------------------------------------------------------------------------- +# LCED source table fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def facility_header(): + return pd.DataFrame({ + "patient_id": ["P001", "P002", "P001"], + "dxver": ["0", "9", "0"], + "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], + "dx1": ["E11.9", "I10", "E11.9"], + "dx2": ["I10", None, None], + "dx3": [None, None, None], + "proc1": ["99213", None, "99213"], + "proc2": [None, "93000", None], + "proc3": [None, None, None], + "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], + "cob": [0.0, 10.0, 0.0], + "coins": [20.0, 0.0, 20.0], + "copay": [30.0, 15.0, 30.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + }) + + +@pytest.fixture() +def inpatient_admissions(): + return pd.DataFrame({ + "patient_id": ["P001", "P003"], + "dxver": ["0", None], + "admdate": ["2020-04-01", "2020-05-15"], + "pdx": ["J18.9", "E10.9"], + "dx1": ["I10", None], + "dx2": [None, None], + "pproc": ["27447", None], + "proc1": [None, "99232"], + "dobyr": [1960, 1980], + "sex": ["M", "M"], + }) + + +@pytest.fixture() +def inpatient_services(): + return pd.DataFrame({ + "patient_id": ["P001", "P002"], + "dxver": ["0", "0"], + "svcdate": ["2020-04-02", "2020-04-10"], + "pdx": ["J18.9", "K92.1"], + "dx1": ["I10", None], + "dx2": [None, None], + "proctyp": ["ICD", "CPT"], + "proc1": ["99232", "44950"], + "cob": [5.0, 0.0], + "coins": [10.0, 25.0], + "copay": [20.0, 40.0], + "dobyr": [1960, 1970], + "sex": ["M", "F"], + }) + + +@pytest.fixture() +def lab_results(): + return pd.DataFrame({ + "patient_id": ["P001", "P002", "P003"], + "dxver": [None, "0", None], + "svcdate": ["2020-01-20", "2020-02-10", "2020-03-05"], + "dx1": ["V58.67", "E11.9", None], + "proctyp": ["CPT", "CPT", "CPT"], + "proc1": ["83036", "83036", None], + "result": ["7.2", "6.8", "5.4"], + "resltcat": [None, None, "normal"], + "resunit": ["mmol/L", "mmol/L", "mmol/L"], + "loinccd": ["14749-6", "14749-6", "14749-6"], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + }) + + +@pytest.fixture() +def outpatient_services(): + return pd.DataFrame({ + "patient_id": ["P002", "P003"], + "dxver": [None, "0"], + "svcdate": ["2020-06-01", "2020-07-01"], + "dx1": ["E10.9", "I10"], + "dx2": [None, None], + "proctyp": ["CPT", "CPT"], + "proc1": ["99213", "99214"], + "cob": [0.0, 0.0], + "coins": [15.0, 20.0], + "copay": [25.0, 35.0], + "dobyr": [1970, 1980], + "sex": ["F", "M"], + }) + + +@pytest.fixture() +def v_drug(): + return pd.DataFrame({ + "patient_id": ["P001", "P002", "P001"], + "prescription_date": ["2020-01-10", "2020-02-05", "2020-01-10"], + "start_date": ["2020-01-15", "2020-02-10", "2020-01-15"], + "end_date": ["2020-04-15", "2020-05-10", "2020-04-15"], + "rx_cui": ["723", "4815", "723"], # metformin, insulin glargine + }) + + +@pytest.fixture() +def v_outpatient_drug_claims(): + return pd.DataFrame({ + "patient_id": ["P001", "P003", "P002"], + "svcdate": ["2020-01-20", "2020-03-01", "2020-02-15"], + "daysupp": [30, 90, 30], + "refill": [0, 1, 0], + "ndcnum": ["00071015523", "00310751030", "00065063136"], + "cob": [0.0, 5.0, 0.0], + "coins": [10.0, 0.0, 20.0], + "copay": [5.0, 10.0, 15.0], + }) + + +@pytest.fixture() +def v_observation(): + return pd.DataFrame({ + "patient_id": ["P001", "P002"], + "observation_date": ["2020-01-15", "2020-02-01"], + "std_value": ["7.2", "6.8"], + "std_uom": ["mmol/L", "mmol/L"], + "loinc_test_id": ["14749-6", "4548-4"], + }) + + +@pytest.fixture() +def v_habit(): + return pd.DataFrame({ + "patient_id": ["P001", "P001", "P002", "P003"], + "mapped_question_answer": ["current smoker", None, "non-smoker", "former smoker"], + "encounter_join_id": [101, 102, 103, 104], + }) + + +@pytest.fixture() +def v_encounter(): + return pd.DataFrame({ + "encounter_join_id": [101, 102, 103, 104], + "encounter_date": ["2020-01-15", "2020-02-01", "2020-02-10", "2020-03-01"], + }) + + +@pytest.fixture() +def v_annual_summary_enrollment(): + return pd.DataFrame({ + "patient_id": ["P001", "P002", "P003"], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + }) + + +@pytest.fixture() +def v_detail_enrollment(): + return pd.DataFrame({ + "patient_id": ["P001", "P001", "P002"], + "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], + "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], + "plantyp": [10, 10, 20], + "rx": ["Y", "Y", "N"], + "hlthplan": ["BlueCross", "BlueCross", "Aetna"], + "dobyr": [1960, 1960, 1970], + "sex": ["M", "M", "F"], + }) + + +# =========================================================================== +# LOINC vocab tests +# =========================================================================== + + +class TestLoadLoincMap: + def test_returns_dataframe(self, loinc_map): + assert isinstance(loinc_map, pd.DataFrame) + + def test_columns_include_loinc_and_component(self, loinc_map): + assert "loinc" in loinc_map.columns + assert "component" in loinc_map.columns + + def test_row_count(self, loinc_map): + assert len(loinc_map) == 10 + + def test_known_component(self, loinc_map): + row = loinc_map[loinc_map["loinc"] == "14749-6"] + assert row["component"].iloc[0] == "Glucose" + + def test_loinc_is_string(self, loinc_map): + assert loinc_map["loinc"].dtype == object + + def test_long_common_name_preserved(self, loinc_map): + assert "long_common_name" in loinc_map.columns + + def test_loinc_num_alias(self, tmp_path): + csv = tmp_path / "loinc.csv" + csv.write_text("loinc_num,component\n14749-6,Glucose\n") + df = load_loinc_map(csv) + assert "loinc" in df.columns + assert "loinc_num" not in df.columns + + +class TestJoinComponentByLoinc: + def test_matched_row_gets_component(self, loinc_map): + df = pd.DataFrame({"patient_id": ["P1"], "loinc": ["14749-6"]}) + result = join_component_by_loinc(df, loinc_map) + assert result.loc[0, "component"] == "Glucose" + + def test_unmatched_row_gets_nan(self, loinc_map): + df = pd.DataFrame({"patient_id": ["P1"], "loinc": ["99999-9"]}) + result = join_component_by_loinc(df, loinc_map) + assert pd.isna(result.loc[0, "component"]) + + def test_row_count_preserved(self, loinc_map): + df = pd.DataFrame({"loinc": ["14749-6", "99999-9", "4548-4"]}) + result = join_component_by_loinc(df, loinc_map) + assert len(result) == 3 + + def test_custom_loinc_column(self, loinc_map): + df = pd.DataFrame({"lab_code": ["14749-6"]}) + result = join_component_by_loinc(df, loinc_map, loinc_col="lab_code") + assert result.loc[0, "component"] == "Glucose" + + def test_does_not_mutate_input(self, loinc_map): + df = pd.DataFrame({"loinc": ["14749-6"]}) + _ = join_component_by_loinc(df, loinc_map) + assert "component" not in df.columns + + +# =========================================================================== +# ICD vocab tests +# =========================================================================== + + +class TestIcdVocab: + def test_icd9_prefixes_contains_e_and_v(self): + assert "E" in ICD9_PREFIXES + assert "V" in ICD9_PREFIXES + + def test_classify_e_prefix_returns_9(self): + s = pd.Series(["E930.0", "E10.9"]) + result = classify_icd_version(s) + assert list(result) == ["9", "9"] + + def test_classify_v_prefix_returns_9(self): + s = pd.Series(["V58.67"]) + result = classify_icd_version(s) + assert result.iloc[0] == "9" + + def test_classify_other_prefix_returns_none(self): + s = pd.Series(["I10", "J18.9", "K92.1"]) + result = classify_icd_version(s) + assert result.isna().all() + + def test_classify_preserves_index(self): + s = pd.Series(["E930.0", "I10"], index=[10, 20]) + result = classify_icd_version(s) + assert list(result.index) == [10, 20] + + +# =========================================================================== +# LCED adapter tests +# =========================================================================== + + +class TestLcedBuildDiagnosis: + def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert DIAGNOSIS.validate(result) == [] + + def test_patient_id_already_named(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_no_null_dx_codes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert result["dx"].notna().all() + + def test_five_sources_contribute(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + # P003 only appears in inpatient_admissions and outpatient_services + assert "P003" in result["patient_id"].values + + def test_v_code_from_lab_results_inferred_icd9(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + v_rows = result[(result["patient_id"] == "P001") & (result["dx"] == "V58.67")] + assert (v_rows["dxver"] == "9").all() + + def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert list(result["patient_id"]) == sorted(result["patient_id"]) + + +class TestLcedBuildTherapy: + def test_schema_valid(self, v_drug, v_outpatient_drug_claims, ndc_map, rxcui_map): + result = build_therapy(v_drug, v_outpatient_drug_claims, ndc_map=ndc_map, rxcui_map=rxcui_map) + assert THERAPY.validate(result) == [] + + def test_v_drug_has_prescription_and_start_date(self, v_drug, v_outpatient_drug_claims): + result = build_therapy(v_drug, v_outpatient_drug_claims) + drug_rows = result[result["rxcui"].notna()] + assert drug_rows["prescription_date"].notna().any() + assert drug_rows["start_date"].notna().any() + + def test_claims_has_fill_date_and_ndc11(self, v_drug, v_outpatient_drug_claims): + result = build_therapy(v_drug, v_outpatient_drug_claims) + claims_rows = result[result["ndc11"].notna()] + assert claims_rows["fill_date"].notna().any() + + def test_end_date_calculated_for_claims(self, v_drug, v_outpatient_drug_claims): + result = build_therapy(v_drug, v_outpatient_drug_claims) + claims_p001 = result[(result["patient_id"] == "P001") & result["ndc11"].notna()] + # fill_date 2020-01-20 + 30 days = 2020-02-19 + assert claims_p001["end_date"].iloc[0] == pd.Timestamp("2020-02-19") + + def test_rxcui_ingredient_joined(self, v_drug, v_outpatient_drug_claims, rxcui_map): + result = build_therapy(v_drug, v_outpatient_drug_claims, rxcui_map=rxcui_map) + metformin_rows = result[result["rxcui"] == "723"] + assert (metformin_rows["ingredient"] == "metformin").all() + + def test_ndc_ingredient_joined(self, v_drug, v_outpatient_drug_claims, ndc_map): + result = build_therapy(v_drug, v_outpatient_drug_claims, ndc_map=ndc_map) + ndc_rows = result[result["ndc11"] == "00071015523"] + assert (ndc_rows["ingredient"] == "metformin").all() + + def test_duplicate_drug_rows_removed(self, v_drug, v_outpatient_drug_claims): + # v_drug has P001 twice with identical row + result = build_therapy(v_drug, v_outpatient_drug_claims) + p001_drug = result[(result["patient_id"] == "P001") & result["rxcui"].notna()] + assert len(p001_drug) == 1 + + def test_both_sources_contribute(self, v_drug, v_outpatient_drug_claims): + result = build_therapy(v_drug, v_outpatient_drug_claims) + assert result["rxcui"].notna().any() + assert result["ndc11"].notna().any() + + def test_no_vocab_gives_null_ingredient(self, v_drug, v_outpatient_drug_claims): + result = build_therapy(v_drug, v_outpatient_drug_claims) + assert result["ingredient"].isna().all() + + +class TestLcedBuildLabtest: + def test_schema_valid(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + assert LABTEST.validate(result) == [] + + def test_both_sources_contribute(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + # v_observation provides P001 and P002; lab_results also provides P003 + assert "P003" in result["patient_id"].values + + def test_observation_loinc_mapped(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + obs_rows = result[result["patient_id"] == "P001"] + assert "14749-6" in obs_rows["loinc"].values + + def test_lab_results_valuecat_mapped(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + # lab_results P003 has resltcat=normal + p003_rows = result[result["patient_id"] == "P003"] + assert "normal" in p003_rows["valuecat"].values + + def test_eventdate_is_datetime(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_no_duplicate_rows(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, v_observation, lab_results): + result = build_labtest(v_observation, lab_results) + assert list(result["patient_id"]) == sorted(result["patient_id"]) + + +class TestLcedBuildProcedure: + def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert PROCEDURE.validate(result) == [] + + def test_no_null_proc_codes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert result["proc"].notna().all() + + def test_lab_results_source_contributes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + # lab_results has proc1=83036 for P001 and P002 + assert "83036" in result["proc"].values + + def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + assert not result.duplicated().any() + + +class TestLcedBuildHabit: + def test_schema_valid(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + assert HABIT.validate(result) == [] + + def test_null_answers_filtered_out(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + assert result["mapped_question_answer"].notna().all() + + def test_encounter_date_joined(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + p001_rows = result[result["patient_id"] == "P001"] + assert pd.api.types.is_datetime64_any_dtype(result["encounter_date"]) + assert p001_rows["encounter_date"].iloc[0] == pd.Timestamp("2020-01-15") + + def test_encounter_join_id_dropped(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + assert "encounter_join_id" not in result.columns + + def test_columns_are_canonical(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + assert list(result.columns) == ["patient_id", "encounter_date", "mapped_question_answer"] + + def test_correct_row_count(self, v_habit, v_encounter): + # 4 rows in v_habit, 1 has null answer → 3 rows + result = build_habit(v_habit, v_encounter) + assert len(result) == 3 + + def test_sorted_by_patient_date_answer(self, v_habit, v_encounter): + result = build_habit(v_habit, v_encounter) + assert list(result["patient_id"]) == sorted(result["patient_id"]) + + +class TestLcedBuildPatinfo: + def test_schema_valid(self, v_annual_summary_enrollment): + result = build_patinfo(v_annual_summary_enrollment) + assert PATINFO.validate(result) == [] + + def test_only_canonical_three_columns(self, v_annual_summary_enrollment, facility_header): + result = build_patinfo(v_annual_summary_enrollment, facility_header) + # LCED patinfo only carries patient_id, dobyr, sex (no extra regional cols) + assert set(result.columns) == {"patient_id", "dobyr", "sex"} + + def test_deduplicates_across_sources(self, v_annual_summary_enrollment, facility_header): + result = build_patinfo(v_annual_summary_enrollment, facility_header) + assert not result.duplicated().any() + + def test_all_patients_present(self, v_annual_summary_enrollment): + result = build_patinfo(v_annual_summary_enrollment) + assert set(result["patient_id"]) == {"P001", "P002", "P003"} + + def test_dobyr_nullable_int(self, v_annual_summary_enrollment): + result = build_patinfo(v_annual_summary_enrollment) + assert result["dobyr"].dtype.name == "Int64" + + +class TestLcedBuildInsurance: + def test_schema_valid(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) + assert INSURANCE.validate(result) == [] + + def test_patient_id_is_string(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) + assert result["patient_id"].dtype == object + + def test_svcdate_is_datetime(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) + assert pd.api.types.is_datetime64_any_dtype(result["svcdate"]) + + def test_no_duplicates(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) + assert not result.duplicated().any() + + +class TestLcedBuildProvider: + def test_schema_valid(self, v_detail_enrollment): + result = build_provider(v_detail_enrollment) + assert PROVIDER.validate(result) == [] + + def test_date_columns_parsed(self, v_detail_enrollment): + result = build_provider(v_detail_enrollment) + assert pd.api.types.is_datetime64_any_dtype(result["dtstart"]) + assert pd.api.types.is_datetime64_any_dtype(result["dtend"]) + + def test_no_duplicates(self, v_detail_enrollment): + result = build_provider(v_detail_enrollment) + assert not result.duplicated().any() + + def test_correct_row_count(self, v_detail_enrollment): + result = build_provider(v_detail_enrollment) + assert len(result) == 3 + + def test_minimal_input_no_optional_cols(self): + df = pd.DataFrame({"patient_id": ["P001", "P002"]}) + result = build_provider(df) + assert "patient_id" in result.columns diff --git a/tests/io/test_source_marketscan.py b/tests/io/test_source_marketscan.py new file mode 100644 index 00000000..429f36c6 --- /dev/null +++ b/tests/io/test_source_marketscan.py @@ -0,0 +1,485 @@ +"""Tests for the IBM MarketScan adapter. + +All source DataFrames are constructed inline to mirror the minimum columns +present in the actual MarketScan PostgreSQL views. Only a handful of rows +are needed per table to exercise the ETL logic. +""" + +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.adapters.marketscan import ( + build_diagnosis, + build_insurance, + build_patinfo, + build_procedure, + build_provider, + build_therapy, +) +from ehrdata.io.source.schema import DIAGNOSIS, INSURANCE, PATINFO, PROCEDURE, PROVIDER, THERAPY +from ehrdata.io.source.vocab.ndc import load_ndc_ingredient_map + +VOCAB_DIR = Path("tests/data/source_vocab") + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def ndc_map(): + return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") + + +@pytest.fixture() +def facility_header(): + return pd.DataFrame({ + "enrolid": [1001, 1002, 1001], + "dxver": ["0", "9", "0"], + "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], + "dx1": ["E11.9", "I10", "E11.9"], + "dx2": ["I10", None, None], + "dx3": [None, None, None], + "proc1": ["99213", None, "99213"], + "proc2": [None, "93000", None], + "proc3": [None, None, None], + "cob": [0.0, 10.0, 0.0], + "coins": [20.0, 0.0, 20.0], + "copay": [30.0, 15.0, 30.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5001], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "NE"], + "msa": [10, 20, 10], + "wgtkey": [1, 2, 1], + "eeclass": ["A", "B", "A"], + "eestatu": ["1", "2", "1"], + "egeoloc": ["11", "22", "11"], + "emprel": ["1", "2", "1"], + "indstry": ["001", "002", "001"], + }) + + +@pytest.fixture() +def inpatient_admissions(): + return pd.DataFrame({ + "enrolid": [1001, 1003], + "dxver": ["0", None], + "admdate": ["2020-04-01", "2020-05-15"], + "pdx": ["J18.9", "E10.9"], + "dx1": ["I10", None], + "dx2": [None, None], + "pproc": ["27447", None], + "proc1": [None, "99232"], + "dobyr": [1960, 1980], + "sex": ["M", "M"], + "efamid": [5001, 5003], + "year": [2020, 2020], + "region": ["NE", "MW"], + "msa": [10, 30], + "wgtkey": [1, 3], + "eeclass": ["A", "C"], + "eestatu": ["1", "3"], + "egeoloc": ["11", "33"], + "emprel": ["1", "3"], + "indstry": ["001", "003"], + }) + + +@pytest.fixture() +def inpatient_services(): + return pd.DataFrame({ + "enrolid": [1001, 1002], + "dxver": ["0", "0"], + "svcdate": ["2020-04-02", "2020-04-10"], + "pdx": ["J18.9", "K92.1"], + "dx1": ["I10", None], + "dx2": [None, None], + "proctyp": ["ICD", "CPT"], + "proc1": ["99232", "44950"], + "cob": [5.0, 0.0], + "coins": [10.0, 25.0], + "copay": [20.0, 40.0], + "dobyr": [1960, 1970], + "sex": ["M", "F"], + "efamid": [5001, 5002], + "year": [2020, 2020], + "region": ["NE", "SE"], + "msa": [10, 20], + "wgtkey": [1, 2], + "eeclass": ["A", "B"], + "eestatu": ["1", "2"], + "egeoloc": ["11", "22"], + "emprel": ["1", "2"], + "indstry": ["001", "002"], + }) + + +@pytest.fixture() +def outpatient_services(): + return pd.DataFrame({ + "enrolid": [1002, 1003], + "dxver": [None, "0"], + "svcdate": ["2020-06-01", "2020-07-01"], + "dx1": ["E10.9", "I10"], + "dx2": [None, None], + "proctyp": ["CPT", "CPT"], + "proc1": ["99213", "99214"], + "cob": [0.0, 0.0], + "coins": [15.0, 20.0], + "copay": [25.0, 35.0], + "dobyr": [1970, 1980], + "sex": ["F", "M"], + "efamid": [5002, 5003], + "year": [2020, 2020], + "region": ["SE", "MW"], + "msa": [20, 30], + "wgtkey": [2, 3], + "eeclass": ["B", "C"], + "eestatu": ["2", "3"], + "egeoloc": ["22", "33"], + "emprel": ["2", "3"], + "indstry": ["002", "003"], + }) + + +@pytest.fixture() +def outpatient_prescription_drugs(): + return pd.DataFrame({ + "enrolid": [1001, 1002, 1001], + "svcdate": ["2020-01-20", "2020-02-05", "2020-01-20"], + "daysupp": [30, 90, 30], + "refill": [0, 1, 0], + "ndcnum": ["00071015523", "00310751030", "00071015523"], # metformin, insulin glargine + "cob": [0.0, 5.0, 0.0], + "coins": [10.0, 0.0, 10.0], + "copay": [5.0, 10.0, 5.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5001], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "NE"], + "msa": [10, 20, 10], + "wgtkey": [1, 2, 1], + "eeclass": ["A", "B", "A"], + "eestatu": ["1", "2", "1"], + "egeoloc": ["11", "22", "11"], + "emprel": ["1", "2", "1"], + "indstry": ["001", "002", "001"], + }) + + +@pytest.fixture() +def enrollment_annual_summary(): + return pd.DataFrame({ + "enrolid": [1001, 1002, 1003], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5003], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "MW"], + "msa": [10, 20, 30], + "wgtkey": [1, 2, 3], + "eeclass": ["A", "B", "C"], + "eestatu": ["1", "2", "3"], + "egeoloc": ["11", "22", "33"], + "emprel": ["1", "2", "3"], + "indstry": ["001","002","003"], + }) + + +@pytest.fixture() +def enrollment_detail(): + return pd.DataFrame({ + "enrolid": [1001, 1001, 1002], + "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], + "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], + "plantyp": [10, 10, 20], + "rx": ["Y", "Y", "N"], + "hlthplan": ["BlueCross", "BlueCross", "Aetna"], + "dobyr": [1960, 1960, 1970], + "sex": ["M", "M", "F"], + "efamid": [5001, 5001, 5002], + "year": [2020, 2021, 2020], + "region": ["NE", "NE", "SE"], + "msa": [10, 10, 20], + "wgtkey": [1, 1, 2], + "eeclass": ["A", "A", "B"], + "eestatu": ["1", "1", "2"], + "egeoloc": ["11", "11", "22"], + "emprel": ["1", "1", "2"], + "indstry": ["001", "001", "002"], + }) + + +# --------------------------------------------------------------------------- +# build_diagnosis +# --------------------------------------------------------------------------- + + +class TestBuildDiagnosis: + def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert DIAGNOSIS.validate(result) == [] + + def test_patient_id_is_string(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert result["patient_id"].dtype == object + + def test_eventdate_is_datetime(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_no_null_dx_codes(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert result["dx"].notna().all() + + def test_enrolid_renamed_to_patient_id(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + patients = list(result["patient_id"]) + assert patients == sorted(patients) + + def test_icd_version_inferred_for_missing(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + # outpatient_services has dxver=None for enrolid=1002 with dx=E10.9 + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + e10_rows = result[(result["patient_id"] == "1002") & (result["dx"] == "E10.9")] + # E prefix + null dxver → inferred as "9" + assert (e10_rows["dxver"] == "9").all() + + def test_wide_dx_unnested(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + # facility_header row 0 has dx1=E11.9 and dx2=I10 → both should appear + patient_1001_dx = set(result[result["patient_id"] == "1001"]["dx"].tolist()) + assert "E11.9" in patient_1001_dx + assert "I10" in patient_1001_dx + + def test_all_four_sources_contribute(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + # Patient 1003 only appears in inpatient_admissions and outpatient_services + assert "1003" in result["patient_id"].values + + +# --------------------------------------------------------------------------- +# build_therapy +# --------------------------------------------------------------------------- + + +class TestBuildTherapy: + def test_schema_valid(self, outpatient_prescription_drugs, ndc_map): + result = build_therapy(outpatient_prescription_drugs, ndc_map=ndc_map) + assert THERAPY.validate(result) == [] + + def test_fill_date_is_svcdate(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert pd.api.types.is_datetime64_any_dtype(result["fill_date"]) + assert result["fill_date"].iloc[0] == pd.Timestamp("2020-01-20") + + def test_end_date_calculated(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + # fill_date 2020-01-20 + 30 days = 2020-02-19 + row = result[result["patient_id"] == "1001"].iloc[0] + assert row["end_date"] == pd.Timestamp("2020-02-19") + + def test_prescription_date_is_nat(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert result["prescription_date"].isna().all() + + def test_start_date_is_nat(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert result["start_date"].isna().all() + + def test_rxcui_is_none(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert result["rxcui"].isna().all() + + def test_ndc11_zero_padded(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert result["ndc11"].str.len().eq(11).all() + + def test_ingredient_joined_from_ndc_map(self, outpatient_prescription_drugs, ndc_map): + result = build_therapy(outpatient_prescription_drugs, ndc_map=ndc_map) + metformin_rows = result[result["ndc11"] == "00071015523"] + assert (metformin_rows["ingredient"] == "metformin").all() + + def test_ingredient_none_without_ndc_map(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert result["ingredient"].isna().all() + + def test_duplicate_removed(self, outpatient_prescription_drugs, ndc_map): + # Source has enrolid=1001 twice with same svcdate/ndcnum/daysupp/refill + result = build_therapy(outpatient_prescription_drugs, ndc_map=ndc_map) + assert len(result[result["patient_id"] == "1001"]) == 1 + + def test_enrolid_renamed_to_patient_id(self, outpatient_prescription_drugs): + result = build_therapy(outpatient_prescription_drugs) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + +# --------------------------------------------------------------------------- +# build_procedure +# --------------------------------------------------------------------------- + + +class TestBuildProcedure: + def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert PROCEDURE.validate(result) == [] + + def test_no_null_proc_codes(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert result["proc"].notna().all() + + def test_enrolid_renamed(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_eventdate_is_datetime(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_proctype_null_for_facility_header(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + # facility_header contributions should have null proctype + fh_date = pd.Timestamp("2020-01-15") + fh_rows = result[(result["patient_id"] == "1001") & (result["eventdate"] == fh_date)] + assert fh_rows["proctype"].isna().all() + + def test_proctype_set_for_outpatient_services(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + op_rows = result[result["patient_id"] == "1002"] + cpt_rows = op_rows[op_rows["proctype"] == "CPT"] + assert len(cpt_rows) > 0 + + def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + assert not result.duplicated().any() + + def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) + patients = list(result["patient_id"]) + assert patients == sorted(patients) + + +# --------------------------------------------------------------------------- +# build_patinfo +# --------------------------------------------------------------------------- + + +class TestBuildPatinfo: + def test_schema_valid(self, enrollment_annual_summary, enrollment_detail): + result = build_patinfo(enrollment_annual_summary, enrollment_detail) + assert PATINFO.validate(result) == [] + + def test_enrolid_renamed(self, enrollment_annual_summary): + result = build_patinfo(enrollment_annual_summary) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_deduplicates_across_sources(self, enrollment_annual_summary, facility_header): + # enrollment_annual_summary and facility_header both contain enrolid=1001 + # with identical (dobyr, sex, efamid, year, ...) → full-row dedup collapses them + result = build_patinfo(enrollment_annual_summary, facility_header) + assert not result.duplicated().any() + + def test_extra_marketscan_columns_present(self, enrollment_annual_summary): + result = build_patinfo(enrollment_annual_summary) + assert "region" in result.columns + assert "year" in result.columns + + def test_dobyr_nullable_int(self, enrollment_annual_summary): + result = build_patinfo(enrollment_annual_summary) + assert result["dobyr"].dtype.name == "Int64" + + def test_single_source(self, enrollment_annual_summary): + result = build_patinfo(enrollment_annual_summary) + assert len(result) == 3 # 3 unique patients in fixture + + def test_all_patients_present(self, enrollment_annual_summary, facility_header): + result = build_patinfo(enrollment_annual_summary, facility_header) + pids = set(result["patient_id"].tolist()) + assert "1001" in pids and "1002" in pids + + +# --------------------------------------------------------------------------- +# build_insurance +# --------------------------------------------------------------------------- + + +class TestBuildInsurance: + def test_schema_valid(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + assert INSURANCE.validate(result) == [] + + def test_enrolid_renamed(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_svcdate_is_datetime(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + assert pd.api.types.is_datetime64_any_dtype(result["svcdate"]) + + def test_cob_coins_copay_numeric(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + for col in ("cob", "coins", "copay"): + assert pd.api.types.is_float_dtype(result[col]) + + def test_no_duplicates(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): + result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + assert not result.duplicated().any() + + +# --------------------------------------------------------------------------- +# build_provider +# --------------------------------------------------------------------------- + + +class TestBuildProvider: + def test_schema_valid(self, enrollment_detail): + result = build_provider(enrollment_detail) + assert PROVIDER.validate(result) == [] + + def test_enrolid_renamed(self, enrollment_detail): + result = build_provider(enrollment_detail) + assert "patient_id" in result.columns + assert "enrolid" not in result.columns + + def test_date_columns_parsed(self, enrollment_detail): + result = build_provider(enrollment_detail) + assert pd.api.types.is_datetime64_any_dtype(result["dtstart"]) + assert pd.api.types.is_datetime64_any_dtype(result["dtend"]) + + def test_correct_row_count(self, enrollment_detail): + # 3 rows in fixture, all distinct + result = build_provider(enrollment_detail) + assert len(result) == 3 + + def test_hlthplan_preserved(self, enrollment_detail): + result = build_provider(enrollment_detail) + assert "BlueCross" in result["hlthplan"].values + + def test_no_duplicates(self, enrollment_detail): + result = build_provider(enrollment_detail) + assert not result.duplicated().any() + + def test_minimal_input_no_optional_cols(self): + # Adapter must not crash if optional columns are absent + df = pd.DataFrame({"enrolid": [1001, 1002]}) + result = build_provider(df) + assert "patient_id" in result.columns + assert len(result) == 2 diff --git a/tests/io/test_source_normalize.py b/tests/io/test_source_normalize.py new file mode 100644 index 00000000..47ce10ee --- /dev/null +++ b/tests/io/test_source_normalize.py @@ -0,0 +1,334 @@ +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.normalize import ( + coerce_date, + coerce_patient_id, + deduplicate, + infer_icd_version, + normalize_diagnosis, + normalize_labtest, + normalize_procedure, + normalize_therapy, + sort_events, +) +from ehrdata.io.source.schema import DIAGNOSIS, LABTEST, PROCEDURE, THERAPY + +FIXTURE_DIR = Path("tests/data/source_basic") + + +# --------------------------------------------------------------------------- +# coerce_patient_id +# --------------------------------------------------------------------------- + + +class TestCoercePatientId: + def test_strips_whitespace(self): + s = pd.Series([" 1 ", " 2", "3 "]) + result = coerce_patient_id(s) + assert list(result) == ["1", "2", "3"] + + def test_converts_int_to_str(self): + s = pd.Series([1, 2, 3]) + result = coerce_patient_id(s) + assert result.dtype == object + assert list(result) == ["1", "2", "3"] + + def test_preserves_index(self): + s = pd.Series(["a", "b"], index=[10, 20]) + result = coerce_patient_id(s) + assert list(result.index) == [10, 20] + + def test_large_int_ids(self): + s = pd.Series([123456789012345]) + result = coerce_patient_id(s) + assert result[0] == "123456789012345" + + +# --------------------------------------------------------------------------- +# coerce_date +# --------------------------------------------------------------------------- + + +class TestCoerceDate: + def test_iso_dates(self): + s = pd.Series(["2020-01-15", "2021-06-30"]) + result = coerce_date(s) + assert pd.api.types.is_datetime64_any_dtype(result) + assert result[0] == pd.Timestamp("2020-01-15") + + def test_invalid_becomes_nat(self): + s = pd.Series(["not-a-date", "2020-01-01"]) + result = coerce_date(s) + assert pd.isna(result[0]) + assert result[1] == pd.Timestamp("2020-01-01") + + def test_already_datetime_passthrough(self): + s = pd.to_datetime(pd.Series(["2020-01-01"])) + result = coerce_date(s) + assert pd.api.types.is_datetime64_any_dtype(result) + + def test_explicit_format(self): + s = pd.Series(["15/01/2020", "01/06/2021"]) + result = coerce_date(s, formats=["%d/%m/%Y"]) + assert result[0] == pd.Timestamp("2020-01-15") + assert result[1] == pd.Timestamp("2021-06-01") + + def test_fallback_when_format_fails(self): + # format list has a wrong entry first, auto-inference picks up the date + s = pd.Series(["2020-01-15"]) + result = coerce_date(s, formats=["%Y/%m/%d", "%Y-%m-%d"]) + assert result[0] == pd.Timestamp("2020-01-15") + + def test_all_null_returns_nat_series(self): + s = pd.Series([None, None]) + result = coerce_date(s) + assert result.isna().all() + + +# --------------------------------------------------------------------------- +# infer_icd_version +# --------------------------------------------------------------------------- + + +class TestInferIcdVersion: + def _make_df(self, dxver_values, dx_values): + return pd.DataFrame({"patient_id": range(len(dx_values)), "dxver": dxver_values, "dx": dx_values}) + + def test_e_prefix_inferred_as_icd9(self): + df = self._make_df([None], ["E930.0"]) + result = infer_icd_version(df) + assert result.loc[0, "dxver"] == "9" + + def test_v_prefix_inferred_as_icd9(self): + df = self._make_df([None], ["V58.67"]) + result = infer_icd_version(df) + assert result.loc[0, "dxver"] == "9" + + def test_existing_dxver_not_overwritten(self): + df = self._make_df(["0"], ["E11.9"]) + result = infer_icd_version(df) + assert result.loc[0, "dxver"] == "0" + + def test_non_ev_prefix_stays_null(self): + df = self._make_df([None], ["I10"]) + result = infer_icd_version(df) + assert pd.isna(result.loc[0, "dxver"]) + + def test_does_not_mutate_input(self): + df = self._make_df([None], ["E930.0"]) + _ = infer_icd_version(df) + assert pd.isna(df.loc[0, "dxver"]) + + def test_mixed_rows(self): + df = self._make_df([None, "0", None, None], ["E930.0", "E11.9", "V58.67", "I10"]) + result = infer_icd_version(df) + assert result.loc[0, "dxver"] == "9" # E prefix + null → inferred + assert result.loc[1, "dxver"] == "0" # explicit, not overwritten + assert result.loc[2, "dxver"] == "9" # V prefix + null → inferred + assert pd.isna(result.loc[3, "dxver"]) # I prefix + null → stays null + + def test_custom_column_names(self): + df = pd.DataFrame({"pid": [1], "ver": [None], "code": ["E930.0"]}) + result = infer_icd_version(df, dx_col="code", dxver_col="ver") + assert result.loc[0, "ver"] == "9" + + +# --------------------------------------------------------------------------- +# deduplicate +# --------------------------------------------------------------------------- + + +class TestDeduplicate: + def test_removes_exact_duplicates(self): + df = pd.DataFrame({"a": [1, 1, 2], "b": ["x", "x", "y"]}) + result = deduplicate(df) + assert len(result) == 2 + + def test_subset_dedup(self): + df = pd.DataFrame({"a": [1, 1, 2], "b": ["x", "z", "y"]}) + result = deduplicate(df, subset=["a"]) + assert len(result) == 2 + + def test_index_reset(self): + df = pd.DataFrame({"a": [1, 1, 2]}, index=[10, 10, 20]) + result = deduplicate(df) + assert list(result.index) == [0, 1] + + def test_no_duplicates_unchanged(self): + df = pd.DataFrame({"a": [1, 2, 3]}) + result = deduplicate(df) + assert len(result) == 3 + + +# --------------------------------------------------------------------------- +# sort_events +# --------------------------------------------------------------------------- + + +class TestSortEvents: + def test_sorted_by_patient_then_date(self): + df = pd.DataFrame({ + "patient_id": ["2", "1", "1"], + "eventdate": pd.to_datetime(["2020-02-01", "2020-03-01", "2020-01-01"]), + }) + result = sort_events(df) + assert list(result["patient_id"]) == ["1", "1", "2"] + assert list(result["eventdate"]) == [pd.Timestamp("2020-01-01"), pd.Timestamp("2020-03-01"), pd.Timestamp("2020-02-01")] + + def test_nat_placed_last(self): + df = pd.DataFrame({ + "patient_id": ["1", "1"], + "eventdate": pd.to_datetime([None, "2020-01-01"]), + }) + result = sort_events(df) + assert result["eventdate"].iloc[0] == pd.Timestamp("2020-01-01") + assert pd.isna(result["eventdate"].iloc[1]) + + def test_custom_column_names(self): + df = pd.DataFrame({ + "pid": ["b", "a"], + "dt": pd.to_datetime(["2020-02-01", "2020-01-01"]), + }) + result = sort_events(df, patient_col="pid", date_col="dt") + assert result["pid"].iloc[0] == "a" + + +# --------------------------------------------------------------------------- +# normalize_diagnosis (composite pipeline) +# --------------------------------------------------------------------------- + + +class TestNormalizeDiagnosis: + @pytest.fixture() + def raw_df(self): + return pd.read_csv(FIXTURE_DIR / "diagnosis.csv") + + def test_schema_valid_after_normalize(self, raw_df): + result = normalize_diagnosis(raw_df) + errors = DIAGNOSIS.validate(result) + assert errors == [], errors + + def test_patient_id_is_stripped_string(self, raw_df): + result = normalize_diagnosis(raw_df) + assert result["patient_id"].dtype == object + assert " " not in result["patient_id"].iloc[-1] # " 4 " → "4" + + def test_eventdate_is_datetime(self, raw_df): + result = normalize_diagnosis(raw_df) + assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) + + def test_icd_version_inferred(self, raw_df): + result = normalize_diagnosis(raw_df) + # Row with dx=E10.9 had null dxver → should be inferred as "9" + e10_rows = result[result["dx"] == "E10.9"] + assert (e10_rows["dxver"] == "9").all() + + def test_duplicate_removed(self, raw_df): + # CSV has patient_id=1 / E11.9 / 2020-01-15 twice + result = normalize_diagnosis(raw_df) + dupes = result[(result["patient_id"] == "1") & (result["dx"] == "E11.9")] + assert len(dupes) == 1 + + def test_sorted_by_patient_then_date(self, raw_df): + result = normalize_diagnosis(raw_df) + patients = list(result["patient_id"]) + assert patients == sorted(patients) + + def test_dxver_column_added_when_absent(self): + df = pd.DataFrame({"patient_id": ["1"], "eventdate": ["2020-01-01"], "dx": ["I10"]}) + result = normalize_diagnosis(df) + assert "dxver" in result.columns + + def test_does_not_mutate_input(self, raw_df): + original_len = len(raw_df) + normalize_diagnosis(raw_df) + assert len(raw_df) == original_len + + +# --------------------------------------------------------------------------- +# normalize_therapy +# --------------------------------------------------------------------------- + + +class TestNormalizeTherapy: + @pytest.fixture() + def raw_df(self): + return pd.read_csv(FIXTURE_DIR / "therapy.csv") + + def test_schema_valid_after_normalize(self, raw_df): + result = normalize_therapy(raw_df) + errors = THERAPY.validate(result) + assert errors == [], errors + + def test_date_columns_parsed(self, raw_df): + result = normalize_therapy(raw_df) + assert pd.api.types.is_datetime64_any_dtype(result["fill_date"]) + assert pd.api.types.is_datetime64_any_dtype(result["end_date"]) + + def test_missing_date_becomes_nat(self, raw_df): + result = normalize_therapy(raw_df) + # row 2 has no end_date + row2 = result[result["patient_id"] == "2"] + assert pd.isna(row2["end_date"].iloc[0]) + + def test_duplicate_removed(self, raw_df): + result = normalize_therapy(raw_df) + assert len(result) == 2 # one duplicate row removed + + def test_patient_id_is_string(self, raw_df): + result = normalize_therapy(raw_df) + assert result["patient_id"].dtype == object + + +# --------------------------------------------------------------------------- +# normalize_labtest +# --------------------------------------------------------------------------- + + +class TestNormalizeLabtest: + @pytest.fixture() + def raw_df(self): + return pd.read_csv(FIXTURE_DIR / "labtest.csv") + + def test_schema_valid_after_normalize(self, raw_df): + result = normalize_labtest(raw_df) + errors = LABTEST.validate(result) + assert errors == [], errors + + def test_duplicate_removed(self, raw_df): + # CSV has patient_id=2 / high duplicate + result = normalize_labtest(raw_df) + assert len(result) == 3 + + def test_sorted_by_patient_then_date(self, raw_df): + result = normalize_labtest(raw_df) + pids = list(result["patient_id"]) + assert pids == sorted(pids) + + +# --------------------------------------------------------------------------- +# normalize_procedure +# --------------------------------------------------------------------------- + + +class TestNormalizeProcedure: + @pytest.fixture() + def raw_df(self): + return pd.read_csv(FIXTURE_DIR / "procedure.csv") + + def test_schema_valid_after_normalize(self, raw_df): + result = normalize_procedure(raw_df) + errors = PROCEDURE.validate(result) + assert errors == [], errors + + def test_duplicate_removed(self, raw_df): + result = normalize_procedure(raw_df) + assert len(result) == 3 # one duplicate removed + + def test_null_proctype_preserved(self, raw_df): + result = normalize_procedure(raw_df) + null_proc = result[result["patient_id"] == "3"] + assert pd.isna(null_proc["proctype"].iloc[0]) diff --git a/tests/io/test_source_schema.py b/tests/io/test_source_schema.py new file mode 100644 index 00000000..b74563e0 --- /dev/null +++ b/tests/io/test_source_schema.py @@ -0,0 +1,116 @@ +import pandas as pd +import pytest + +from ehrdata.io.source.schema import ( + ALL_SCHEMAS, + DIAGNOSIS, + HABIT, + INSURANCE, + LABTEST, + PATINFO, + PROCEDURE, + PROVIDER, + THERAPY, + ColumnSpec, + TableSchema, +) + + +class TestColumnSpec: + def test_defaults(self): + col = ColumnSpec("patient_id", "object") + assert col.name == "patient_id" + assert col.dtype == "object" + assert col.nullable is True + + def test_non_nullable(self): + col = ColumnSpec("dx", "object", nullable=False) + assert col.nullable is False + + def test_frozen(self): + col = ColumnSpec("x", "object") + with pytest.raises(Exception): + col.name = "y" # type: ignore[misc] + + +class TestTableSchema: + def test_column_names(self): + assert DIAGNOSIS.column_names == ["patient_id", "dxver", "eventdate", "dx"] + + def test_empty_returns_zero_row_dataframe(self): + df = DIAGNOSIS.empty() + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + assert list(df.columns) == DIAGNOSIS.column_names + + def test_empty_dtype_object(self): + df = PATINFO.empty() + assert df["patient_id"].dtype == object + assert df["sex"].dtype == object + + def test_empty_datetime_dtype(self): + df = DIAGNOSIS.empty() + assert pd.api.types.is_datetime64_any_dtype(df["eventdate"]) + + def test_validate_valid_dataframe(self): + df = pd.DataFrame({"patient_id": ["1"], "dxver": ["0"], "eventdate": pd.to_datetime(["2020-01-01"]), "dx": ["E11.9"]}) + assert DIAGNOSIS.validate(df) == [] + + def test_validate_missing_column(self): + df = pd.DataFrame({"patient_id": ["1"], "dxver": ["0"], "eventdate": pd.to_datetime(["2020-01-01"])}) + errors = DIAGNOSIS.validate(df) + assert len(errors) == 1 + assert "dx" in errors[0] + + def test_validate_multiple_missing_columns(self): + df = pd.DataFrame({"patient_id": ["1"]}) + errors = DIAGNOSIS.validate(df) + assert len(errors) == 3 + + def test_validate_strict_rejects_extra_columns(self): + df = pd.DataFrame({ + "patient_id": ["1"], "dxver": ["0"], + "eventdate": pd.to_datetime(["2020-01-01"]), + "dx": ["E11.9"], "extra_col": [99], + }) + assert DIAGNOSIS.validate(df, strict=False) == [] + errors = DIAGNOSIS.validate(df, strict=True) + assert len(errors) == 1 + assert "extra_col" in errors[0] + + def test_validate_strict_no_false_positives(self): + df = DIAGNOSIS.empty() + assert DIAGNOSIS.validate(df, strict=True) == [] + + +class TestAllSchemas: + def test_contains_all_eight_tables(self): + assert set(ALL_SCHEMAS.keys()) == {"diagnosis", "therapy", "labtest", "procedure", "patinfo", "insurance", "provider", "habit"} + + def test_all_schemas_start_with_patient_id(self): + for name, schema in ALL_SCHEMAS.items(): + assert schema.column_names[0] == "patient_id", f"{name} does not start with patient_id" + + def test_all_schemas_have_non_empty_columns(self): + for name, schema in ALL_SCHEMAS.items(): + assert len(schema.columns) > 0, f"{name} has no columns" + + @pytest.mark.parametrize("schema,expected_cols", [ + (THERAPY, ["patient_id", "prescription_date", "start_date", "fill_date", "end_date", "refill", "rxcui", "ndc11", "ingredient"]), + (LABTEST, ["patient_id", "eventdate", "value", "valuecat", "unit", "loinc"]), + (PROCEDURE, ["patient_id", "proctype", "eventdate", "proc"]), + (PATINFO, ["patient_id", "dobyr", "sex"]), + (INSURANCE, ["patient_id", "svcdate", "cob", "coins", "copay"]), + (PROVIDER, ["patient_id", "dtstart", "dtend", "plantyp", "rx", "hlthplan"]), + (HABIT, ["patient_id", "encounter_date", "mapped_question_answer"]), + ]) + def test_schema_columns(self, schema, expected_cols): + assert schema.column_names == expected_cols + + def test_schema_lookup_by_name(self): + assert ALL_SCHEMAS["diagnosis"] is DIAGNOSIS + assert ALL_SCHEMAS["therapy"] is THERAPY + + def test_schemas_are_frozen(self): + with pytest.raises(Exception): + DIAGNOSIS.name = "other" # type: ignore[misc] diff --git a/tests/io/test_source_to_ehrdata.py b/tests/io/test_source_to_ehrdata.py new file mode 100644 index 00000000..a4b0729b --- /dev/null +++ b/tests/io/test_source_to_ehrdata.py @@ -0,0 +1,311 @@ +"""Tests for src/ehrdata/io/source/to_ehrdata.py. + +Verifies that the source-layer canonical DataFrames are correctly assembled +into EHRData objects: obs, var, X shape and contents, uns provenance. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from ehrdata.io.source.to_ehrdata import to_ehrdata + + +# --------------------------------------------------------------------------- +# Minimal fixture helpers +# --------------------------------------------------------------------------- + + +def _patinfo(*patient_ids: str) -> pd.DataFrame: + return pd.DataFrame({ + "patient_id": list(patient_ids), + "dobyr": [1960 + i for i in range(len(patient_ids))], + "sex": ["F" if i % 2 == 0 else "M" for i in range(len(patient_ids))], + }) + + +def _diagnosis(rows: list[tuple[str, str]]) -> pd.DataFrame: + pids, dxs = zip(*rows) if rows else ([], []) + return pd.DataFrame({ + "patient_id": list(pids), + "dxver": [None] * len(pids), + "eventdate": pd.NaT, + "dx": list(dxs), + }) + + +def _therapy(rows: list[tuple[str, str | None]]) -> pd.DataFrame: + pids, ings = zip(*rows) if rows else ([], []) + return pd.DataFrame({ + "patient_id": list(pids), + "prescription_date": pd.NaT, + "start_date": pd.NaT, + "fill_date": pd.NaT, + "end_date": pd.NaT, + "refill": pd.array([pd.NA] * len(pids), dtype="Int64"), + "rxcui": None, + "ndc11": None, + "ingredient": list(ings), + }) + + +def _labtest(rows: list[tuple[str, str | None]]) -> pd.DataFrame: + pids, loincs = zip(*rows) if rows else ([], []) + return pd.DataFrame({ + "patient_id": list(pids), + "eventdate": pd.NaT, + "value": None, + "valuecat": None, + "unit": None, + "loinc": list(loincs), + }) + + +def _procedure(rows: list[tuple[str, str]]) -> pd.DataFrame: + pids, procs = zip(*rows) if rows else ([], []) + return pd.DataFrame({ + "patient_id": list(pids), + "proctype": None, + "eventdate": pd.NaT, + "proc": list(procs), + }) + + +# --------------------------------------------------------------------------- +# TestObsPopulation +# --------------------------------------------------------------------------- + + +class TestObsPopulation: + def test_obs_index_is_patient_id(self): + edata = to_ehrdata(_patinfo("P1", "P2", "P3")) + assert set(edata.obs_names) == {"P1", "P2", "P3"} + + def test_obs_index_dtype_is_string(self): + edata = to_ehrdata(_patinfo("P1", "P2")) + assert edata.obs.index.dtype == object + + def test_obs_contains_dobyr_and_sex(self): + edata = to_ehrdata(_patinfo("P1")) + assert "dobyr" in edata.obs.columns + assert "sex" in edata.obs.columns + + def test_obs_row_count(self): + edata = to_ehrdata(_patinfo("P1", "P2", "P3")) + assert edata.n_obs == 3 + + def test_duplicate_patients_in_patinfo_deduplicated(self): + patinfo = pd.concat([_patinfo("P1"), _patinfo("P1")], ignore_index=True) + edata = to_ehrdata(patinfo) + assert edata.n_obs == 1 + + def test_patients_not_in_patinfo_excluded_from_X(self): + patinfo = _patinfo("P1") + dx = _diagnosis([("P1", "E11.9"), ("GHOST", "I10")]) + edata = to_ehrdata(patinfo, diagnosis=dx) + assert "P1" in edata.obs_names + assert "GHOST" not in edata.obs_names + + +# --------------------------------------------------------------------------- +# TestVarPopulation +# --------------------------------------------------------------------------- + + +class TestVarPopulation: + def test_empty_tables_gives_empty_var(self): + edata = to_ehrdata(_patinfo("P1")) + assert edata.n_vars == 0 + + def test_diagnosis_concepts_prefixed(self): + dx = _diagnosis([("P1", "E11.9"), ("P1", "I10")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert "diagnosis:E11.9" in edata.var_names + assert "diagnosis:I10" in edata.var_names + + def test_therapy_concepts_prefixed(self): + th = _therapy([("P1", "metformin")]) + edata = to_ehrdata(_patinfo("P1"), therapy=th) + assert "therapy:metformin" in edata.var_names + + def test_labtest_concepts_prefixed(self): + lb = _labtest([("P1", "14749-6")]) + edata = to_ehrdata(_patinfo("P1"), labtest=lb) + assert "labtest:14749-6" in edata.var_names + + def test_procedure_concepts_prefixed(self): + pr = _procedure([("P1", "99213")]) + edata = to_ehrdata(_patinfo("P1"), procedure=pr) + assert "procedure:99213" in edata.var_names + + def test_var_has_concept_source_column(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert "concept_source" in edata.var.columns + + def test_var_has_concept_code_column(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert "concept_code" in edata.var.columns + + def test_concept_source_value(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + row = edata.var.loc["diagnosis:E11.9"] + assert row["concept_source"] == "diagnosis" + assert row["concept_code"] == "E11.9" + + def test_null_loinc_excluded_from_var(self): + lb = _labtest([("P1", None), ("P1", "14749-6")]) + edata = to_ehrdata(_patinfo("P1"), labtest=lb) + assert edata.n_vars == 1 + assert "labtest:14749-6" in edata.var_names + + def test_null_ingredient_excluded_from_var(self): + th = _therapy([("P1", None), ("P1", "aspirin")]) + edata = to_ehrdata(_patinfo("P1"), therapy=th) + assert edata.n_vars == 1 + assert "therapy:aspirin" in edata.var_names + + def test_concepts_from_multiple_tables_unioned(self): + dx = _diagnosis([("P1", "E11.9")]) + th = _therapy([("P1", "metformin")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx, therapy=th) + assert "diagnosis:E11.9" in edata.var_names + assert "therapy:metformin" in edata.var_names + + def test_var_index_is_sorted(self): + dx = _diagnosis([("P1", "Z99"), ("P1", "A01")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert list(edata.var_names) == sorted(edata.var_names.tolist()) + + +# --------------------------------------------------------------------------- +# TestXMatrix +# --------------------------------------------------------------------------- + + +class TestXMatrix: + def test_x_shape(self): + dx = _diagnosis([("P1", "E11.9"), ("P2", "I10")]) + edata = to_ehrdata(_patinfo("P1", "P2"), diagnosis=dx) + assert edata.X.shape == (2, 2) + + def test_x_dtype_is_float64(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert edata.X.dtype == np.float64 + + def test_x_binary_values_only(self): + dx = _diagnosis([("P1", "E11.9"), ("P1", "I10"), ("P2", "E11.9")]) + edata = to_ehrdata(_patinfo("P1", "P2"), diagnosis=dx) + unique_vals = np.unique(edata.X) + assert set(unique_vals.tolist()).issubset({0.0, 1.0}) + + def test_x_is_one_when_patient_has_concept(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1", "P2"), diagnosis=dx) + p1_idx = edata.obs_names.tolist().index("P1") + c_idx = edata.var_names.tolist().index("diagnosis:E11.9") + assert edata.X[p1_idx, c_idx] == 1.0 + + def test_x_is_zero_when_patient_lacks_concept(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1", "P2"), diagnosis=dx) + p2_idx = edata.obs_names.tolist().index("P2") + c_idx = edata.var_names.tolist().index("diagnosis:E11.9") + assert edata.X[p2_idx, c_idx] == 0.0 + + def test_duplicate_events_do_not_inflate_x(self): + # P1 has E11.9 three times — X should still be 1.0 + dx = _diagnosis([("P1", "E11.9"), ("P1", "E11.9"), ("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert edata.X[0, 0] == 1.0 + + def test_no_event_tables_gives_zero_column_x(self): + edata = to_ehrdata(_patinfo("P1", "P2")) + assert edata.X.shape == (2, 0) + + def test_x_empty_when_no_patients(self): + patinfo = pd.DataFrame({"patient_id": [], "dobyr": [], "sex": []}) + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(patinfo, diagnosis=dx) + assert edata.n_obs == 0 + + def test_x_rows_align_with_obs(self): + dx = _diagnosis([("P2", "E11.9")]) + edata = to_ehrdata(_patinfo("P1", "P2"), diagnosis=dx) + p1_idx = edata.obs_names.tolist().index("P1") + p2_idx = edata.obs_names.tolist().index("P2") + c_idx = edata.var_names.tolist().index("diagnosis:E11.9") + assert edata.X[p1_idx, c_idx] == 0.0 + assert edata.X[p2_idx, c_idx] == 1.0 + + +# --------------------------------------------------------------------------- +# TestUns +# --------------------------------------------------------------------------- + + +class TestUns: + def test_source_stored_in_uns(self): + edata = to_ehrdata(_patinfo("P1"), source="marketscan") + assert edata.uns["source_io_source"] == "marketscan" + + def test_source_none_when_not_provided(self): + edata = to_ehrdata(_patinfo("P1")) + assert edata.uns["source_io_source"] is None + + def test_tables_used_diagnosis(self): + dx = _diagnosis([("P1", "E11.9")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx) + assert "diagnosis" in edata.uns["source_io_tables"] + + def test_tables_used_therapy(self): + th = _therapy([("P1", "metformin")]) + edata = to_ehrdata(_patinfo("P1"), therapy=th) + assert "therapy" in edata.uns["source_io_tables"] + + def test_tables_used_lists_all_provided(self): + dx = _diagnosis([("P1", "E11.9")]) + th = _therapy([("P1", "metformin")]) + lb = _labtest([("P1", "14749-6")]) + edata = to_ehrdata(_patinfo("P1"), diagnosis=dx, therapy=th, labtest=lb) + assert set(edata.uns["source_io_tables"]) == {"diagnosis", "therapy", "labtest"} + + def test_tables_used_empty_when_no_event_tables(self): + edata = to_ehrdata(_patinfo("P1")) + assert edata.uns["source_io_tables"] == [] + + +# --------------------------------------------------------------------------- +# TestReturnType +# --------------------------------------------------------------------------- + + +class TestReturnType: + def test_returns_ehrdata(self): + from ehrdata import EHRData + + edata = to_ehrdata(_patinfo("P1")) + assert isinstance(edata, EHRData) + + def test_full_pipeline_with_all_tables(self): + from ehrdata import EHRData + + patinfo = _patinfo("P1", "P2", "P3") + dx = _diagnosis([("P1", "E11.9"), ("P2", "I10"), ("P3", "E11.9")]) + th = _therapy([("P1", "metformin"), ("P2", None)]) + lb = _labtest([("P3", "14749-6"), ("P1", None)]) + pr = _procedure([("P2", "99213")]) + + edata = to_ehrdata(patinfo, diagnosis=dx, therapy=th, labtest=lb, procedure=pr, source="test") + + assert isinstance(edata, EHRData) + assert edata.n_obs == 3 + # concepts: diagnosis:E11.9, diagnosis:I10, therapy:metformin, labtest:14749-6, procedure:99213 + assert edata.n_vars == 5 + assert edata.uns["source_io_source"] == "test" + assert set(edata.uns["source_io_tables"]) == {"diagnosis", "therapy", "labtest", "procedure"} diff --git a/tests/io/test_source_vocab.py b/tests/io/test_source_vocab.py new file mode 100644 index 00000000..c5c3f359 --- /dev/null +++ b/tests/io/test_source_vocab.py @@ -0,0 +1,157 @@ +from pathlib import Path + +import pandas as pd +import pytest + +from ehrdata.io.source.vocab.ndc import join_ingredient_by_ndc, load_ndc_ingredient_map +from ehrdata.io.source.vocab.rxnorm import join_ingredient_by_rxcui, load_rxcui_ingredient_map + +VOCAB_DIR = Path("tests/data/source_vocab") + + +# --------------------------------------------------------------------------- +# NDC +# --------------------------------------------------------------------------- + + +class TestLoadNdcIngredientMap: + @pytest.fixture() + def ndc_map(self): + return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") + + def test_returns_dataframe(self, ndc_map): + assert isinstance(ndc_map, pd.DataFrame) + + def test_columns(self, ndc_map): + assert list(ndc_map.columns) == ["ndc11", "rxcui", "ingredient"] + + def test_row_count(self, ndc_map): + assert len(ndc_map) == 10 + + def test_ndc11_zero_padded(self, ndc_map): + # All ndc11 values must be exactly 11 characters + assert ndc_map["ndc11"].str.len().eq(11).all() + + def test_ndc11_is_string(self, ndc_map): + assert ndc_map["ndc11"].dtype == object + + def test_known_ingredient(self, ndc_map): + row = ndc_map[ndc_map["ndc11"] == "00071015523"] + assert row["ingredient"].iloc[0] == "metformin" + + def test_leading_zeros_preserved(self, ndc_map): + # 00065063136 starts with two zeros — must not be dropped + assert "00065063136" in ndc_map["ndc11"].values + + def test_accepts_string_path(self): + df = load_ndc_ingredient_map(str(VOCAB_DIR / "ndc_ingredient_map.txt")) + assert len(df) == 10 + + +class TestJoinIngredientByNdc: + @pytest.fixture() + def ndc_map(self): + return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") + + def test_matched_row_gets_ingredient(self, ndc_map): + df = pd.DataFrame({"patient_id": ["1"], "ndc11": ["00071015523"]}) + result = join_ingredient_by_ndc(df, ndc_map) + assert result.loc[0, "ingredient"] == "metformin" + + def test_unmatched_row_gets_nan(self, ndc_map): + df = pd.DataFrame({"patient_id": ["1"], "ndc11": ["99999999999"]}) + result = join_ingredient_by_ndc(df, ndc_map) + assert pd.isna(result.loc[0, "ingredient"]) + + def test_existing_ingredient_column_overwritten(self, ndc_map): + df = pd.DataFrame({"patient_id": ["1"], "ndc11": ["00071015523"], "ingredient": ["old"]}) + result = join_ingredient_by_ndc(df, ndc_map) + assert result.loc[0, "ingredient"] == "metformin" + + def test_row_count_preserved(self, ndc_map): + df = pd.DataFrame({"patient_id": ["1", "2", "3"], "ndc11": ["00071015523", "99999999999", "00310751030"]}) + result = join_ingredient_by_ndc(df, ndc_map) + assert len(result) == 3 + + def test_custom_ndc_column_name(self, ndc_map): + df = pd.DataFrame({"patient_id": ["1"], "drug_ndc": ["00071015523"]}) + result = join_ingredient_by_ndc(df, ndc_map, ndc_col="drug_ndc") + assert result.loc[0, "ingredient"] == "metformin" + + def test_does_not_mutate_input(self, ndc_map): + df = pd.DataFrame({"ndc11": ["00071015523"]}) + _ = join_ingredient_by_ndc(df, ndc_map) + assert "ingredient" not in df.columns + + def test_duplicate_ndc_in_map_no_explosion(self, ndc_map): + # Even if map has duplicates, join must not multiply rows + dup_map = pd.concat([ndc_map, ndc_map.head(1)], ignore_index=True) + df = pd.DataFrame({"ndc11": ["00071015523", "00310751030"]}) + result = join_ingredient_by_ndc(df, dup_map) + assert len(result) == 2 + + +# --------------------------------------------------------------------------- +# RxNorm +# --------------------------------------------------------------------------- + + +class TestLoadRxcuiIngredientMap: + @pytest.fixture() + def rxcui_map(self): + return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt") + + def test_returns_dataframe(self, rxcui_map): + assert isinstance(rxcui_map, pd.DataFrame) + + def test_columns(self, rxcui_map): + assert list(rxcui_map.columns) == ["rxcui", "ingredient"] + + def test_v1_index_column_dropped(self, rxcui_map): + assert "v1" not in rxcui_map.columns + + def test_row_count(self, rxcui_map): + assert len(rxcui_map) == 10 + + def test_known_ingredient(self, rxcui_map): + row = rxcui_map[rxcui_map["rxcui"] == "723"] + assert row["ingredient"].iloc[0] == "metformin" + + def test_rxcui_is_string(self, rxcui_map): + assert rxcui_map["rxcui"].dtype == object + + +class TestJoinIngredientByRxcui: + @pytest.fixture() + def rxcui_map(self): + return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt") + + def test_matched_row_gets_ingredient(self, rxcui_map): + df = pd.DataFrame({"patient_id": ["1"], "rxcui": ["723"]}) + result = join_ingredient_by_rxcui(df, rxcui_map) + assert result.loc[0, "ingredient"] == "metformin" + + def test_unmatched_row_gets_nan(self, rxcui_map): + df = pd.DataFrame({"patient_id": ["1"], "rxcui": ["9999999"]}) + result = join_ingredient_by_rxcui(df, rxcui_map) + assert pd.isna(result.loc[0, "ingredient"]) + + def test_existing_ingredient_column_overwritten(self, rxcui_map): + df = pd.DataFrame({"rxcui": ["723"], "ingredient": ["old"]}) + result = join_ingredient_by_rxcui(df, rxcui_map) + assert result.loc[0, "ingredient"] == "metformin" + + def test_row_count_preserved(self, rxcui_map): + df = pd.DataFrame({"rxcui": ["723", "9999999", "4815"]}) + result = join_ingredient_by_rxcui(df, rxcui_map) + assert len(result) == 3 + + def test_custom_rxcui_column_name(self, rxcui_map): + df = pd.DataFrame({"drug_rxcui": ["723"]}) + result = join_ingredient_by_rxcui(df, rxcui_map, rxcui_col="drug_rxcui") + assert result.loc[0, "ingredient"] == "metformin" + + def test_does_not_mutate_input(self, rxcui_map): + df = pd.DataFrame({"rxcui": ["723"]}) + _ = join_ingredient_by_rxcui(df, rxcui_map) + assert "ingredient" not in df.columns From d9536c0ced386bf06761e189a0a640f03bf89fd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:39:45 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/tutorials/source_cprd_overview.ipynb | 222 +++++----- docs/tutorials/source_icd_mapping.ipynb | 159 ++++--- docs/tutorials/source_lced_cohort.ipynb | 211 +++++---- docs/tutorials/source_loinc_mapping.ipynb | 152 ++++--- src/ehrdata/io/source/adapters/cprd.py | 19 +- src/ehrdata/io/source/adapters/lced.py | 75 ++-- src/ehrdata/io/source/adapters/marketscan.py | 70 +-- src/ehrdata/io/source/extract.py | 22 +- src/ehrdata/io/source/normalize.py | 1 - src/ehrdata/io/source/schema.py | 13 +- src/ehrdata/io/source/to_ehrdata.py | 21 +- src/ehrdata/io/source/vocab/icd.py | 11 +- src/ehrdata/io/source/vocab/loinc.py | 5 +- src/ehrdata/io/source/vocab/ndc.py | 5 +- src/ehrdata/io/source/vocab/prodcode.py | 5 +- src/ehrdata/io/source/vocab/readcode.py | 5 +- src/ehrdata/io/source/vocab/rxnorm.py | 5 +- tests/io/test_source_cprd.py | 15 +- tests/io/test_source_extract.py | 16 +- tests/io/test_source_lced.py | 421 ++++++++++-------- tests/io/test_source_marketscan.py | 441 +++++++++++-------- tests/io/test_source_normalize.py | 52 ++- tests/io/test_source_schema.py | 64 ++- tests/io/test_source_to_ehrdata.py | 92 ++-- tests/io/test_source_vocab.py | 8 +- 25 files changed, 1218 insertions(+), 892 deletions(-) diff --git a/docs/tutorials/source_cprd_overview.ipynb b/docs/tutorials/source_cprd_overview.ipynb index 2ba018e7..a195447f 100644 --- a/docs/tutorials/source_cprd_overview.ipynb +++ b/docs/tutorials/source_cprd_overview.ipynb @@ -24,11 +24,6 @@ "metadata": {}, "outputs": [], "source": [ - "import io\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import numpy as np\n", "import pandas as pd\n", "\n", "from ehrdata.io.source.adapters import cprd\n", @@ -69,61 +64,81 @@ "source": [ "# --- Synthetic CPRD DataFrames (mirrors real schema) ---\n", "\n", - "clinical = pd.DataFrame({\n", - " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\"],\n", - " \"eventdate\": [\"01/06/2010\", \"15/08/2012\", \"03/03/2008\", \"22/11/2015\", \"22/11/2015\"],\n", - " \"medcode\": [100, 200, 100, 300, 300],\n", - " \"enttype\": [4, 4, 4, 5, 5],\n", - " \"adid\": [\"A1\", \"A2\", \"A3\", \"A4\", \"A4\"],\n", - "})\n", + "clinical = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\"],\n", + " \"eventdate\": [\"01/06/2010\", \"15/08/2012\", \"03/03/2008\", \"22/11/2015\", \"22/11/2015\"],\n", + " \"medcode\": [100, 200, 100, 300, 300],\n", + " \"enttype\": [4, 4, 4, 5, 5],\n", + " \"adid\": [\"A1\", \"A2\", \"A3\", \"A4\", \"A4\"],\n", + " }\n", + ")\n", "\n", - "referral = pd.DataFrame({\n", - " \"patid\": [\"P002\", \"P003\"],\n", - " \"eventdate\": [\"14/07/2009\", \"01/01/2016\"],\n", - " \"medcode\": [200, 400],\n", - " \"enttype\": [3, 3],\n", - " \"adid\": [\"\", \"\"],\n", - "})\n", + "referral = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P002\", \"P003\"],\n", + " \"eventdate\": [\"14/07/2009\", \"01/01/2016\"],\n", + " \"medcode\": [200, 400],\n", + " \"enttype\": [3, 3],\n", + " \"adid\": [\"\", \"\"],\n", + " }\n", + ")\n", "\n", - "test_data = pd.DataFrame({\n", - " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", - " \"eventdate\": [\"10/05/2011\", \"20/09/2013\", \"01/04/2017\"],\n", - " \"medcode\": [500, 100, 200],\n", - " \"enttype\": [4, 5, 4],\n", - " \"adid\": [\"A5\", \"A6\", \"A7\"],\n", - " \"data1\": [1.0, 2.0, 3.0], \"data2\": [7.1, 6.4, 8.9],\n", - " \"data3\": [\"%\", \"%\", \"%\"], \"data4\": [\"\", \"\", \"\"],\n", - " \"data5\": [None, None, None], \"data6\": [None, None, None], \"data7\": [None, None, None],\n", - "})\n", + "test_data = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"eventdate\": [\"10/05/2011\", \"20/09/2013\", \"01/04/2017\"],\n", + " \"medcode\": [500, 100, 200],\n", + " \"enttype\": [4, 5, 4],\n", + " \"adid\": [\"A5\", \"A6\", \"A7\"],\n", + " \"data1\": [1.0, 2.0, 3.0],\n", + " \"data2\": [7.1, 6.4, 8.9],\n", + " \"data3\": [\"%\", \"%\", \"%\"],\n", + " \"data4\": [\"\", \"\", \"\"],\n", + " \"data5\": [None, None, None],\n", + " \"data6\": [None, None, None],\n", + " \"data7\": [None, None, None],\n", + " }\n", + ")\n", "\n", - "additional = pd.DataFrame({\n", - " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", - " \"enttype\": [4, 4, 5],\n", - " \"adid\": [\"A1\", \"A3\", \"A4\"],\n", - " \"data1\": [1.0, 1.0, 2.0], \"data2\": [7.1, 6.4, 8.9],\n", - " \"data3\": [\"%\", \"%\", \"%\"], \"data4\": [\"\", \"\", \"\"],\n", - " \"data5\": [None, None, None], \"data6\": [None, None, None], \"data7\": [None, None, None],\n", - "})\n", + "additional = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"enttype\": [4, 4, 5],\n", + " \"adid\": [\"A1\", \"A3\", \"A4\"],\n", + " \"data1\": [1.0, 1.0, 2.0],\n", + " \"data2\": [7.1, 6.4, 8.9],\n", + " \"data3\": [\"%\", \"%\", \"%\"],\n", + " \"data4\": [\"\", \"\", \"\"],\n", + " \"data5\": [None, None, None],\n", + " \"data6\": [None, None, None],\n", + " \"data7\": [None, None, None],\n", + " }\n", + ")\n", "\n", - "therapy_data = pd.DataFrame({\n", - " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\"],\n", - " \"eventdate\": [\"10/01/2010\", \"20/03/2011\", \"05/06/2009\", \"12/12/2014\"],\n", - " \"prodcode\": [\"PROD1\", \"PROD1\", \"PROD2\", \"PROD3\"],\n", - " \"bnfcode\": [\"6.1.2\", \"6.1.2\", \"6.1.1\", \"2.12\"],\n", - " \"qty\": [28, 28, 56, 28],\n", - " \"issueseq\": [1, 2, 1, 1],\n", - "})\n", + "therapy_data = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P001\", \"P001\", \"P002\", \"P003\"],\n", + " \"eventdate\": [\"10/01/2010\", \"20/03/2011\", \"05/06/2009\", \"12/12/2014\"],\n", + " \"prodcode\": [\"PROD1\", \"PROD1\", \"PROD2\", \"PROD3\"],\n", + " \"bnfcode\": [\"6.1.2\", \"6.1.2\", \"6.1.1\", \"2.12\"],\n", + " \"qty\": [28, 28, 56, 28],\n", + " \"issueseq\": [1, 2, 1, 1],\n", + " }\n", + ")\n", "\n", - "patient = pd.DataFrame({\n", - " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", - " \"yob\": [155, 162, 178], # encoded as year_of_birth - 1800\n", - " \"sex\": [\"Female\", \"Male\", \"Female\"],\n", - " \"pracid\": [\"PR1\", \"PR1\", \"PR2\"],\n", - " \"gender\": [2, 1, 2],\n", - " \"frd\": [\"01/01/2000\", \"01/06/1998\", \"15/03/2010\"],\n", - " \"tod\": [None, None, None],\n", - " \"deathdate\": [None, None, None],\n", - "})\n", + "patient = pd.DataFrame(\n", + " {\n", + " \"patid\": [\"P001\", \"P002\", \"P003\"],\n", + " \"yob\": [155, 162, 178], # encoded as year_of_birth - 1800\n", + " \"sex\": [\"Female\", \"Male\", \"Female\"],\n", + " \"pracid\": [\"PR1\", \"PR1\", \"PR2\"],\n", + " \"gender\": [2, 1, 2],\n", + " \"frd\": [\"01/01/2000\", \"01/06/1998\", \"15/03/2010\"],\n", + " \"tod\": [None, None, None],\n", + " \"deathdate\": [None, None, None],\n", + " }\n", + ")\n", "\n", "print(f\"Clinical rows : {len(clinical)}\")\n", "print(f\"Referral rows : {len(referral)}\")\n", @@ -176,20 +191,25 @@ ], "source": [ "# Synthetic vocabulary maps (in practice, load from medical.txt / product.csv)\n", - "medical_map_df = pd.DataFrame({\n", - " \"medcode\": [100, 200, 300, 400, 500],\n", - " \"readcode\": [\"A10..00\", \"C10..00\", \"E12..00\", \"F45..00\", \"44J3.00\"],\n", - " \"desc\": [\"Cholera\", \"Diabetes mellitus\", \"Obesity\", \"Anxiety disorder\", \"HbA1c\"],\n", - "})\n", + "medical_map_df = pd.DataFrame(\n", + " {\n", + " \"medcode\": [100, 200, 300, 400, 500],\n", + " \"readcode\": [\"A10..00\", \"C10..00\", \"E12..00\", \"F45..00\", \"44J3.00\"],\n", + " \"desc\": [\"Cholera\", \"Diabetes mellitus\", \"Obesity\", \"Anxiety disorder\", \"HbA1c\"],\n", + " }\n", + ")\n", "\n", - "product_map_df = pd.DataFrame({\n", - " \"prodcode\": [\"PROD1\", \"PROD2\", \"PROD3\"],\n", - " \"drugsubstance\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", - " \"drugsubstance.updated\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", - "})\n", + "product_map_df = pd.DataFrame(\n", + " {\n", + " \"prodcode\": [\"PROD1\", \"PROD2\", \"PROD3\"],\n", + " \"drugsubstance\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", + " \"drugsubstance.updated\": [\"metformin\", \"insulin glargine\", \"atorvastatin\"],\n", + " }\n", + ")\n", "\n", "# Use the loader functions on in-memory CSV to demonstrate the API\n", - "import tempfile, os\n", + "import tempfile\n", + "import os\n", "\n", "with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".txt\", delete=False) as f:\n", " medical_map_df.to_csv(f, sep=\"\\t\", index=False)\n", @@ -200,7 +220,7 @@ " prod_path = f.name\n", "\n", "medical_map = readcode.load_medical_map(med_path)\n", - "product_map = prodcode.load_product_map(prod_path)\n", + "product_map = prodcode.load_product_map(prod_path)\n", "\n", "os.unlink(med_path)\n", "os.unlink(prod_path)\n", @@ -738,6 +758,7 @@ "source": [ "_DATE_FMT = \"%d/%m/%Y\"\n", "\n", + "\n", "def _patid_dates(df, date_col=\"eventdate\"):\n", " return (\n", " df[[\"patid\", date_col]]\n", @@ -746,16 +767,22 @@ " .drop_duplicates()\n", " )\n", "\n", - "all_events = pd.concat([\n", - " _patid_dates(clinical),\n", - " _patid_dates(referral),\n", - " _patid_dates(test_data),\n", - " _patid_dates(therapy_data),\n", - "], ignore_index=True).drop_duplicates()\n", + "\n", + "all_events = pd.concat(\n", + " [\n", + " _patid_dates(clinical),\n", + " _patid_dates(referral),\n", + " _patid_dates(test_data),\n", + " _patid_dates(therapy_data),\n", + " ],\n", + " ignore_index=True,\n", + ").drop_duplicates()\n", "\n", "print(f\"Total unique (patid, eventdate) pairs : {len(all_events):,}\")\n", "print(f\"Unique patients : {all_events['patid'].nunique():,}\")\n", - "print(f\"Date range : {all_events['eventdate'].min().date()} → {all_events['eventdate'].max().date()}\")\n", + "print(\n", + " f\"Date range : {all_events['eventdate'].min().date()} → {all_events['eventdate'].max().date()}\"\n", + ")\n", "all_events.groupby(all_events[\"eventdate\"].dt.year).size().rename(\"events\").to_frame().head(10)" ] }, @@ -863,19 +890,11 @@ "diag_coded[\"readcode_reduced\"] = diag_coded[\"dx\"].str[:3] + \"...\"\n", "\n", "# Observation count per stub\n", - "hierarchy_obs = (\n", - " diag_coded.groupby(\"readcode_reduced\")\n", - " .size()\n", - " .rename(\"obs_count\")\n", - " .sort_values(ascending=False)\n", - ")\n", + "hierarchy_obs = diag_coded.groupby(\"readcode_reduced\").size().rename(\"obs_count\").sort_values(ascending=False)\n", "\n", "# Patient count per stub\n", "hierarchy_pat = (\n", - " diag_coded.groupby(\"readcode_reduced\")[\"patient_id\"]\n", - " .nunique()\n", - " .rename(\"patient_count\")\n", - " .sort_values(ascending=False)\n", + " diag_coded.groupby(\"readcode_reduced\")[\"patient_id\"].nunique().rename(\"patient_count\").sort_values(ascending=False)\n", ")\n", "\n", "pd.concat([hierarchy_obs, hierarchy_pat], axis=1).head(10)" @@ -962,8 +981,7 @@ ], "source": [ "drug_summary = (\n", - " therapy\n", - " .groupby(\"ingredient\", dropna=False)\n", + " therapy.groupby(\"ingredient\", dropna=False)\n", " .agg(\n", " obs_count=(\"patient_id\", \"count\"),\n", " patient_count=(\"patient_id\", \"nunique\"),\n", @@ -1082,20 +1100,22 @@ "_FMT = \"%d/%m/%Y\"\n", "\n", "qp = patient.copy()\n", - "qp[\"frd\"] = pd.to_datetime(qp[\"frd\"], format=_FMT, errors=\"coerce\")\n", - "qp[\"tod\"] = pd.to_datetime(qp[\"tod\"], format=_FMT, errors=\"coerce\")\n", - "qp[\"deathdate\"] = pd.to_datetime(qp[\"deathdate\"], format=_FMT, errors=\"coerce\")\n", - "qp[\"dob\"] = pd.to_datetime((qp[\"yob\"] + 1800).astype(str) + \"-01-01\", errors=\"coerce\")\n", + "qp[\"frd\"] = pd.to_datetime(qp[\"frd\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"tod\"] = pd.to_datetime(qp[\"tod\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"deathdate\"] = pd.to_datetime(qp[\"deathdate\"], format=_FMT, errors=\"coerce\")\n", + "qp[\"dob\"] = pd.to_datetime((qp[\"yob\"] + 1800).astype(str) + \"-01-01\", errors=\"coerce\")\n", "\n", - "frd_ok = qp[\"frd\"].between(\"1918-01-01\", \"2018-12-30\")\n", - "tod_order_ok = (qp[\"frd\"] <= qp[\"tod\"]) | qp[\"tod\"].isna()\n", - "tod_range_ok = qp[\"tod\"].between(\"1918-01-01\", \"2018-12-30\") | qp[\"tod\"].isna()\n", - "death_frd_ok = (qp[\"frd\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", - "death_dob_ok = (qp[\"dob\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", - "yob_ok = (qp[\"yob\"] + 1800) >= 1887\n", - "gender_ok = qp[\"gender\"] != 3\n", + "frd_ok = qp[\"frd\"].between(\"1918-01-01\", \"2018-12-30\")\n", + "tod_order_ok = (qp[\"frd\"] <= qp[\"tod\"]) | qp[\"tod\"].isna()\n", + "tod_range_ok = qp[\"tod\"].between(\"1918-01-01\", \"2018-12-30\") | qp[\"tod\"].isna()\n", + "death_frd_ok = (qp[\"frd\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", + "death_dob_ok = (qp[\"dob\"] < qp[\"deathdate\"]) | qp[\"deathdate\"].isna()\n", + "yob_ok = (qp[\"yob\"] + 1800) >= 1887\n", + "gender_ok = qp[\"gender\"] != 3\n", "\n", - "qualified = qp[frd_ok & tod_order_ok & tod_range_ok & death_frd_ok & death_dob_ok & yob_ok & gender_ok].drop_duplicates(subset=\"patid\")\n", + "qualified = qp[frd_ok & tod_order_ok & tod_range_ok & death_frd_ok & death_dob_ok & yob_ok & gender_ok].drop_duplicates(\n", + " subset=\"patid\"\n", + ")\n", "\n", "print(f\"Total patients : {len(qp):,}\")\n", "print(f\"Qualified patients: {len(qualified):,}\")\n", @@ -1138,9 +1158,9 @@ "\n", "# Restrict to qualified patients\n", "qualified_ids = set(qualified[\"patid\"])\n", - "patinfo_q = patinfo[patinfo[\"patient_id\"].isin(qualified_ids)]\n", - "diag_q = diag[diag[\"patient_id\"].isin(qualified_ids)]\n", - "therapy_q = therapy[therapy[\"patient_id\"].isin(qualified_ids)]\n", + "patinfo_q = patinfo[patinfo[\"patient_id\"].isin(qualified_ids)]\n", + "diag_q = diag[diag[\"patient_id\"].isin(qualified_ids)]\n", + "therapy_q = therapy[therapy[\"patient_id\"].isin(qualified_ids)]\n", "\n", "edata = to_ehrdata(\n", " patinfo_q,\n", diff --git a/docs/tutorials/source_icd_mapping.ipynb b/docs/tutorials/source_icd_mapping.ipynb index 929c59e8..cdad2275 100644 --- a/docs/tutorials/source_icd_mapping.ipynb +++ b/docs/tutorials/source_icd_mapping.ipynb @@ -29,11 +29,7 @@ "outputs": [], "source": [ "import re\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from ehrdata.io.source.adapters import lced\n", - "from ehrdata.io.source import normalize" + "import pandas as pd" ] }, { @@ -160,12 +156,12 @@ ], "source": [ "# Representative diagnosis codes from LCED / MarketScan\n", - "test_codes = pd.DataFrame({\n", - " \"dx\": [\"250.00\", \"E11.9\", \"I25.10\", \"410.00\", \"V45.86\",\n", - " \"428.0\", \"Z79.4\", \"160.1\", \"E11\", \"25000\"],\n", - " \"dxver\": [\"9\", \"0\", \"0\", None, None,\n", - " None, \"0\", None, None, \"9\"],\n", - "})\n", + "test_codes = pd.DataFrame(\n", + " {\n", + " \"dx\": [\"250.00\", \"E11.9\", \"I25.10\", \"410.00\", \"V45.86\", \"428.0\", \"Z79.4\", \"160.1\", \"E11\", \"25000\"],\n", + " \"dxver\": [\"9\", \"0\", \"0\", None, None, None, \"0\", None, None, \"9\"],\n", + " }\n", + ")\n", "test_codes" ] }, @@ -296,8 +292,9 @@ } ], "source": [ - "_ICD9_PATTERN = re.compile(r'^[VvEe]?\\d')\n", - "_ICD10_PATTERN = re.compile(r'^[A-Za-z]\\d')\n", + "_ICD9_PATTERN = re.compile(r\"^[VvEe]?\\d\")\n", + "_ICD10_PATTERN = re.compile(r\"^[A-Za-z]\\d\")\n", + "\n", "\n", "def infer_icd_version(dx, dxver):\n", " \"\"\"Return '9' or '0'; fall back to regex if dxver is null.\"\"\"\n", @@ -312,10 +309,8 @@ " return \"0\"\n", " return None\n", "\n", - "test_codes[\"dxver_inferred\"] = [\n", - " infer_icd_version(row.dx, row.dxver)\n", - " for _, row in test_codes.iterrows()\n", - "]\n", + "\n", + "test_codes[\"dxver_inferred\"] = [infer_icd_version(row.dx, row.dxver) for _, row in test_codes.iterrows()]\n", "test_codes" ] }, @@ -419,11 +414,13 @@ ], "source": [ "# Minimal GEM table (ICD-10 → ICD-9)\n", - "gem_i10 = pd.DataFrame({\n", - " \"icd10\": [\"E119\", \"E11\", \"I2510\", \"I110\", \"I259\", \"I509\", \"G309\"],\n", - " \"icd9\": [\"25000\", \"250\", \"41401\", \"40210\", \"41400\", \"42800\", \"33100\"],\n", - " \"flags\": [\"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\"],\n", - "})\n", + "gem_i10 = pd.DataFrame(\n", + " {\n", + " \"icd10\": [\"E119\", \"E11\", \"I2510\", \"I110\", \"I259\", \"I509\", \"G309\"],\n", + " \"icd9\": [\"25000\", \"250\", \"41401\", \"40210\", \"41400\", \"42800\", \"33100\"],\n", + " \"flags\": [\"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\", \"10000\"],\n", + " }\n", + ")\n", "\n", "# Strip dots from codes before lookup (LCED stores codes without dots)\n", "test_codes[\"dx_nodot\"] = test_codes[\"dx\"].str.replace(\".\", \"\", regex=False)\n", @@ -481,30 +478,41 @@ ], "source": [ "# Minimal ICD-10-CM CCS table (mirrors DXCCSR_v2020-3.CSV structure)\n", - "ccs_icd10 = pd.DataFrame({\n", - " \"icd10\": [\"E119\", \"E118\", \"I509\", \"I2510\", \"I110\", \"G309\", \"N183\", \"J449\", \"C50.9\"],\n", - " \"icd10_desc\": [\"T2DM unspecified\", \"T2DM other\", \"Heart failure\",\n", - " \"Coronary artery disease\", \"Hypertensive HF\", \"Alzheimer disease\",\n", - " \"CKD stage 3\", \"COPD unspecified\", \"Breast cancer\"],\n", - " \"ccs_category\": [\"END002\", \"END002\", \"CIR019\", \"CIR007\", \"CIR007\",\n", - " \"NVS010\", \"GEN003\", \"RSP010\", \"NEO011\"],\n", - " \"ccs_desc\": [\n", - " \"Diabetes mellitus without complication\",\n", - " \"Diabetes mellitus without complication\",\n", - " \"Heart failure\",\n", - " \"Coronary atherosclerosis and other heart disease\",\n", - " \"Coronary atherosclerosis and other heart disease\",\n", - " \"Alzheimer disease and related disorders\",\n", - " \"Chronic kidney disease\",\n", - " \"Chronic obstructive pulmonary disease and bronchiectasis\",\n", - " \"Cancer of breast\",\n", - " ],\n", - "})\n", + "ccs_icd10 = pd.DataFrame(\n", + " {\n", + " \"icd10\": [\"E119\", \"E118\", \"I509\", \"I2510\", \"I110\", \"G309\", \"N183\", \"J449\", \"C50.9\"],\n", + " \"icd10_desc\": [\n", + " \"T2DM unspecified\",\n", + " \"T2DM other\",\n", + " \"Heart failure\",\n", + " \"Coronary artery disease\",\n", + " \"Hypertensive HF\",\n", + " \"Alzheimer disease\",\n", + " \"CKD stage 3\",\n", + " \"COPD unspecified\",\n", + " \"Breast cancer\",\n", + " ],\n", + " \"ccs_category\": [\"END002\", \"END002\", \"CIR019\", \"CIR007\", \"CIR007\", \"NVS010\", \"GEN003\", \"RSP010\", \"NEO011\"],\n", + " \"ccs_desc\": [\n", + " \"Diabetes mellitus without complication\",\n", + " \"Diabetes mellitus without complication\",\n", + " \"Heart failure\",\n", + " \"Coronary atherosclerosis and other heart disease\",\n", + " \"Coronary atherosclerosis and other heart disease\",\n", + " \"Alzheimer disease and related disorders\",\n", + " \"Chronic kidney disease\",\n", + " \"Chronic obstructive pulmonary disease and bronchiectasis\",\n", + " \"Cancer of breast\",\n", + " ],\n", + " }\n", + ")\n", + "\n", "\n", "def search_ccs(df, pattern, col=\"ccs_desc\"):\n", " mask = df[col].str.contains(pattern, case=False, na=False)\n", " return df[mask][[\"ccs_category\", \"ccs_desc\"]].drop_duplicates()\n", "\n", + "\n", "for term in [\"heart failure\", \"diabetes\", \"chronic kidney\", \"cancer\", \"coronary\"]:\n", " hits = search_ccs(ccs_icd10, term)\n", " print(f\"'{term}': {hits['ccs_category'].tolist()}\")" @@ -538,27 +546,41 @@ "source": [ "# ICD-9 codes for Type 2 Diabetes (from lced_template_analysis.R)\n", "T2D_ICD9 = [\n", - " \"250.0\", \"25000\", \"25002\",\n", - " \"2501\", \"25010\", \"25012\",\n", - " \"2504\", \"25040\", \"25042\",\n", - " \"2505\", \"25050\", \"25052\",\n", - " \"2506\", \"25060\", \"25062\",\n", - " \"2507\", \"25070\", \"25072\",\n", - " \"2509\", \"25090\", \"25092\",\n", + " \"250.0\",\n", + " \"25000\",\n", + " \"25002\",\n", + " \"2501\",\n", + " \"25010\",\n", + " \"25012\",\n", + " \"2504\",\n", + " \"25040\",\n", + " \"25042\",\n", + " \"2505\",\n", + " \"25050\",\n", + " \"25052\",\n", + " \"2506\",\n", + " \"25060\",\n", + " \"25062\",\n", + " \"2507\",\n", + " \"25070\",\n", + " \"25072\",\n", + " \"2509\",\n", + " \"25090\",\n", + " \"25092\",\n", "]\n", "# ICD-10 prefix\n", "T2D_ICD10_PREFIX = \"E11\"\n", "\n", "# AMI (Acute Myocardial Infarction)\n", - "AMI_ICD9 = [\"410\"] + [f\"41{x}\" for x in [\"00\",\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\"]]\n", + "AMI_ICD9 = [\"410\"] + [f\"41{x}\" for x in [\"00\", \"01\", \"02\", \"03\", \"04\", \"05\", \"06\", \"07\", \"08\", \"09\"]]\n", "AMI_ICD10_PREFIX = \"I21\"\n", "\n", "# Stroke\n", - "STROKE_ICD9_PREFIXES = [\"160\", \"161\", \"162\", \"163\", \"430\", \"431\", \"432\", \"433\", \"434\", \"435\"]\n", + "STROKE_ICD9_PREFIXES = [\"160\", \"161\", \"162\", \"163\", \"430\", \"431\", \"432\", \"433\", \"434\", \"435\"]\n", "STROKE_ICD10_PREFIXES = [\"I60\", \"I61\", \"I62\", \"I63\", \"I64\"]\n", "\n", "# CHF (Congestive Heart Failure)\n", - "CHF_ICD9 = [\"4280\", \"428\"] + [f\"428{x}\" for x in range(10)]\n", + "CHF_ICD9 = [\"4280\", \"428\"] + [f\"428{x}\" for x in range(10)]\n", "CHF_ICD10_PREFIX = \"I50\"\n", "\n", "print(\"T2D ICD-9 codes:\", T2D_ICD9[:6], \"...\")\n", @@ -601,15 +623,25 @@ ], "source": [ "# Synthetic diagnosis table (mirrors lced.build_diagnosis output)\n", - "diagnosis = pd.DataFrame({\n", - " \"patient_id\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\", \"P004\", \"P004\"],\n", - " \"eventdate\": pd.to_datetime([\n", - " \"2014-03-01\", \"2015-11-15\", \"2016-07-20\",\n", - " \"2013-09-05\", \"2017-04-10\", \"2014-12-01\", \"2018-02-28\",\n", - " ]),\n", - " \"dx\": [\"E119\", \"I509\", \"25000\", \"I2510\", \"41001\", \"410\", \"E119\"],\n", - " \"dxver\": [\"0\", \"0\", \"9\", \"0\", \"9\", \"9\", \"0\"],\n", - "})\n", + "diagnosis = pd.DataFrame(\n", + " {\n", + " \"patient_id\": [\"P001\", \"P001\", \"P002\", \"P003\", \"P003\", \"P004\", \"P004\"],\n", + " \"eventdate\": pd.to_datetime(\n", + " [\n", + " \"2014-03-01\",\n", + " \"2015-11-15\",\n", + " \"2016-07-20\",\n", + " \"2013-09-05\",\n", + " \"2017-04-10\",\n", + " \"2014-12-01\",\n", + " \"2018-02-28\",\n", + " ]\n", + " ),\n", + " \"dx\": [\"E119\", \"I509\", \"25000\", \"I2510\", \"41001\", \"410\", \"E119\"],\n", + " \"dxver\": [\"0\", \"0\", \"9\", \"0\", \"9\", \"9\", \"0\"],\n", + " }\n", + ")\n", + "\n", "\n", "def has_icd(dx_series, icd9_list=None, icd10_prefixes=None, icd9_prefixes=None):\n", " \"\"\"Return a boolean mask for rows matching ICD-9 list or ICD-10 prefix.\"\"\"\n", @@ -627,10 +659,11 @@ " mask |= dx_series.str.startswith(pfx, na=False)\n", " return mask\n", "\n", - "diagnosis[\"t2d\"] = has_icd(diagnosis[\"dx\"], icd9_list=T2D_ICD9, icd10_prefixes=\"E11\")\n", - "diagnosis[\"ami\"] = has_icd(diagnosis[\"dx\"], icd9_list=AMI_ICD9, icd10_prefixes=\"I21\")\n", + "\n", + "diagnosis[\"t2d\"] = has_icd(diagnosis[\"dx\"], icd9_list=T2D_ICD9, icd10_prefixes=\"E11\")\n", + "diagnosis[\"ami\"] = has_icd(diagnosis[\"dx\"], icd9_list=AMI_ICD9, icd10_prefixes=\"I21\")\n", "diagnosis[\"stroke\"] = has_icd(diagnosis[\"dx\"], icd9_prefixes=STROKE_ICD9_PREFIXES, icd10_prefixes=STROKE_ICD10_PREFIXES)\n", - "diagnosis[\"chf\"] = has_icd(diagnosis[\"dx\"], icd9_list=CHF_ICD9, icd10_prefixes=\"I50\")\n", + "diagnosis[\"chf\"] = has_icd(diagnosis[\"dx\"], icd9_list=CHF_ICD9, icd10_prefixes=\"I50\")\n", "\n", "print(diagnosis.to_string(index=False))\n", "print()\n", diff --git a/docs/tutorials/source_lced_cohort.ipynb b/docs/tutorials/source_lced_cohort.ipynb index f337b1bd..d0183bf1 100644 --- a/docs/tutorials/source_lced_cohort.ipynb +++ b/docs/tutorials/source_lced_cohort.ipynb @@ -29,10 +29,7 @@ "outputs": [], "source": [ "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from ehrdata.io.source.adapters import lced\n", - "from ehrdata.io.source.vocab import ndc as ndc_vocab, rxnorm, loinc as loinc_vocab" + "import pandas as pd" ] }, { @@ -67,55 +64,67 @@ "np.random.seed(0)\n", "\n", "N_PAT = 20\n", - "pids = [f\"P{i:04d}\" for i in range(1, N_PAT + 1)]\n", + "pids = [f\"P{i:04d}\" for i in range(1, N_PAT + 1)]\n", "\n", "# --- Enrollment summary ---\n", - "v_annual = pd.DataFrame({\n", - " \"patient_id\": pids,\n", - " \"dobyr\": np.random.randint(1940, 1975, N_PAT),\n", - " \"sex\": np.random.choice([\"M\", \"F\"], N_PAT),\n", - " **{f\"enrind{m}\": np.random.choice([0, 1], N_PAT, p=[0.05, 0.95]) for m in range(1, 13)},\n", - " \"year\": [2015] * N_PAT,\n", - "})\n", + "v_annual = pd.DataFrame(\n", + " {\n", + " \"patient_id\": pids,\n", + " \"dobyr\": np.random.randint(1940, 1975, N_PAT),\n", + " \"sex\": np.random.choice([\"M\", \"F\"], N_PAT),\n", + " **{f\"enrind{m}\": np.random.choice([0, 1], N_PAT, p=[0.05, 0.95]) for m in range(1, 13)},\n", + " \"year\": [2015] * N_PAT,\n", + " }\n", + ")\n", "\n", "# --- Outpatient services (diagnosis) ---\n", "_dx_pool = [\n", - " (\"E119\", \"0\"), (\"E118\", \"0\"), (\"25000\", \"9\"), (\"25002\", \"9\"),\n", - " (\"I509\", \"0\"), (\"41001\", \"9\"), (\"I259\", \"0\"), (\"Z794\", \"0\"),\n", + " (\"E119\", \"0\"),\n", + " (\"E118\", \"0\"),\n", + " (\"25000\", \"9\"),\n", + " (\"25002\", \"9\"),\n", + " (\"I509\", \"0\"),\n", + " (\"41001\", \"9\"),\n", + " (\"I259\", \"0\"),\n", + " (\"Z794\", \"0\"),\n", "]\n", "\n", "rows = []\n", "for pid in pids:\n", " for _ in range(np.random.randint(2, 6)):\n", " dx, dxver = _dx_pool[np.random.randint(len(_dx_pool))]\n", - " rows.append({\"patient_id\": pid,\n", - " \"svcdate\": pd.Timestamp(\"2013-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1000))),\n", - " \"dx1\": dx, \"dxver\": dxver})\n", + " rows.append(\n", + " {\n", + " \"patient_id\": pid,\n", + " \"svcdate\": pd.Timestamp(\"2013-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1000))),\n", + " \"dx1\": dx,\n", + " \"dxver\": dxver,\n", + " }\n", + " )\n", "v_outpatient = pd.DataFrame(rows)\n", "\n", "# --- Drug claims ---\n", "# NDC codes: metformin, GLP-1s, SGLT-2s, DPP-4s, SUs (synthetic 11-digit codes)\n", - "MET_NDCS = [\"00093717401\", \"00093717501\"]\n", - "GLP1_NDCS = [\"00169368812\", \"00169368915\"] # exenatide, liraglutide\n", - "SGLT2_NDCS = [\"00310653090\", \"57844011290\"] # dapagliflozin, canagliflozin\n", - "DPP4_NDCS = [\"00006054054\", \"00310027190\"] # sitagliptin, saxagliptin\n", - "SU_NDCS = [\"00093116401\", \"00093013901\"] # glimepiride, glyburide\n", + "MET_NDCS = [\"00093717401\", \"00093717501\"]\n", + "GLP1_NDCS = [\"00169368812\", \"00169368915\"] # exenatide, liraglutide\n", + "SGLT2_NDCS = [\"00310653090\", \"57844011290\"] # dapagliflozin, canagliflozin\n", + "DPP4_NDCS = [\"00006054054\", \"00310027190\"] # sitagliptin, saxagliptin\n", + "SU_NDCS = [\"00093116401\", \"00093013901\"] # glimepiride, glyburide\n", "\n", "SECOND_LINE = GLP1_NDCS + SGLT2_NDCS + DPP4_NDCS + SU_NDCS\n", - "DRUG_CLASS = (\n", - " {n: \"glp1\" for n in GLP1_NDCS} |\n", - " {n: \"sglt2\" for n in SGLT2_NDCS} |\n", - " {n: \"dpp4\" for n in DPP4_NDCS} |\n", - " {n: \"su\" for n in SU_NDCS}\n", + "DRUG_CLASS = (\n", + " dict.fromkeys(GLP1_NDCS, \"glp1\")\n", + " | dict.fromkeys(SGLT2_NDCS, \"sglt2\")\n", + " | dict.fromkeys(DPP4_NDCS, \"dpp4\")\n", + " | dict.fromkeys(SU_NDCS, \"su\")\n", ")\n", "\n", "drg_rows = []\n", "for pid in np.random.choice(pids, 40, replace=True):\n", " pool = MET_NDCS + SECOND_LINE\n", - " ndc = pool[np.random.randint(len(pool))]\n", + " ndc = pool[np.random.randint(len(pool))]\n", " date = pd.Timestamp(\"2014-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1200)))\n", - " drg_rows.append({\"patient_id\": pid, \"svcdate\": date,\n", - " \"ndcnum\": ndc, \"daysupp\": 30, \"refill\": 0})\n", + " drg_rows.append({\"patient_id\": pid, \"svcdate\": date, \"ndcnum\": ndc, \"daysupp\": 30, \"refill\": 0})\n", "v_drug_claims = pd.DataFrame(drg_rows)\n", "\n", "# --- Observations (HbA1c) ---\n", @@ -124,16 +133,17 @@ "for pid in np.random.choice(pids, 60, replace=True):\n", " loinc = np.random.choice(HBA1C_LOINCS)\n", " value = round(np.random.uniform(5.5, 11.0), 1)\n", - " date = pd.Timestamp(\"2014-06-01\") + pd.Timedelta(days=int(np.random.randint(0, 900)))\n", - " obs_rows.append({\"patient_id\": pid, \"observation_date\": date,\n", - " \"loinc_test_id\": loinc, \"std_value\": value, \"std_uom\": \"%\"})\n", + " date = pd.Timestamp(\"2014-06-01\") + pd.Timedelta(days=int(np.random.randint(0, 900)))\n", + " obs_rows.append(\n", + " {\"patient_id\": pid, \"observation_date\": date, \"loinc_test_id\": loinc, \"std_value\": value, \"std_uom\": \"%\"}\n", + " )\n", "v_observation = pd.DataFrame(obs_rows)\n", "\n", "# --- Inpatient services (MACE outcomes) ---\n", "MACE_CODES = [\"I219\", \"I639\", \"I509\", \"41001\", \"43401\"]\n", - "inp_rows = []\n", + "inp_rows = []\n", "for pid in np.random.choice(pids, 10, replace=True):\n", - " dx = MACE_CODES[np.random.randint(len(MACE_CODES))]\n", + " dx = MACE_CODES[np.random.randint(len(MACE_CODES))]\n", " date = pd.Timestamp(\"2015-01-01\") + pd.Timedelta(days=int(np.random.randint(0, 1000)))\n", " inp_rows.append({\"patient_id\": pid, \"svcdate\": date, \"dx1\": dx, \"dx2\": None})\n", "v_inpatient_services = pd.DataFrame(inp_rows)\n", @@ -172,16 +182,31 @@ } ], "source": [ - "T2D_ICD9 = [\"25000\", \"25002\", \"25010\", \"25012\", \"25040\", \"25042\",\n", - " \"25050\", \"25052\", \"25060\", \"25062\", \"25070\", \"25072\",\n", - " \"25090\", \"25092\"]\n", + "T2D_ICD9 = [\n", + " \"25000\",\n", + " \"25002\",\n", + " \"25010\",\n", + " \"25012\",\n", + " \"25040\",\n", + " \"25042\",\n", + " \"25050\",\n", + " \"25052\",\n", + " \"25060\",\n", + " \"25062\",\n", + " \"25070\",\n", + " \"25072\",\n", + " \"25090\",\n", + " \"25092\",\n", + "]\n", + "\n", "\n", "def is_t2d(dx_series):\n", " nodot = dx_series.str.replace(\".\", \"\", regex=False)\n", - " icd9 = nodot.isin(T2D_ICD9)\n", + " icd9 = nodot.isin(T2D_ICD9)\n", " icd10 = dx_series.str.startswith(\"E11\", na=False)\n", " return icd9 | icd10\n", "\n", + "\n", "t2d_flag = is_t2d(v_outpatient[\"dx1\"])\n", "t2d_patients = v_outpatient[t2d_flag][\"patient_id\"].unique()\n", "print(f\"Patients with ≥1 T2D code: {len(t2d_patients)}\")" @@ -307,18 +332,9 @@ } ], "source": [ - "met_claims = v_drug_claims[\n", - " v_drug_claims[\"patient_id\"].isin(eligible) &\n", - " v_drug_claims[\"ndcnum\"].isin(MET_NDCS)\n", - "].copy()\n", + "met_claims = v_drug_claims[v_drug_claims[\"patient_id\"].isin(eligible) & v_drug_claims[\"ndcnum\"].isin(MET_NDCS)].copy()\n", "\n", - "met_index = (\n", - " met_claims\n", - " .groupby(\"patient_id\")[\"svcdate\"]\n", - " .min()\n", - " .rename(\"metformin_start\")\n", - " .reset_index()\n", - ")\n", + "met_index = met_claims.groupby(\"patient_id\")[\"svcdate\"].min().rename(\"metformin_start\").reset_index()\n", "\n", "print(f\"Patients with metformin: {len(met_index)}\")\n", "met_index.head()" @@ -400,22 +416,21 @@ "a1c = (\n", " v_observation[v_observation[\"loinc_test_id\"].isin(HBA1C_LOINCS)]\n", " .copy()\n", - " .rename(columns={\"observation_date\": \"obs_date\", \"std_value\": \"hba1c\"})\n", - " [[\"patient_id\", \"obs_date\", \"hba1c\"]]\n", + " .rename(columns={\"observation_date\": \"obs_date\", \"std_value\": \"hba1c\"})[[\"patient_id\", \"obs_date\", \"hba1c\"]]\n", ")\n", "\n", "# Merge with metformin index\n", "a1c = a1c.merge(met_index, on=\"patient_id\", how=\"inner\")\n", "\n", "# Apply filters: after metformin start, after study start, HbA1c ≥ 7\n", - "a1c = a1c[\n", - " (a1c[\"obs_date\"] > a1c[\"metformin_start\"]) &\n", - " (a1c[\"obs_date\"] > STUDY_START) &\n", - " (a1c[\"hba1c\"] >= 7.0)\n", - "].sort_values([\"patient_id\", \"obs_date\"]).reset_index(drop=True)\n", + "a1c = (\n", + " a1c[(a1c[\"obs_date\"] > a1c[\"metformin_start\"]) & (a1c[\"obs_date\"] > STUDY_START) & (a1c[\"hba1c\"] >= 7.0)]\n", + " .sort_values([\"patient_id\", \"obs_date\"])\n", + " .reset_index(drop=True)\n", + ")\n", "\n", "# Days between consecutive measurements per patient\n", - "a1c[\"prev_date\"] = a1c.groupby(\"patient_id\")[\"obs_date\"].shift(1)\n", + "a1c[\"prev_date\"] = a1c.groupby(\"patient_id\")[\"obs_date\"].shift(1)\n", "a1c[\"days_since_last\"] = (a1c[\"obs_date\"] - a1c[\"prev_date\"]).dt.days\n", "\n", "# Poor control flag\n", @@ -426,8 +441,7 @@ " a1c[a1c[\"poor_control\"]]\n", " .groupby(\"patient_id\")\n", " .first()\n", - " .reset_index()\n", - " [[\"patient_id\", \"obs_date\", \"hba1c\", \"metformin_start\"]]\n", + " .reset_index()[[\"patient_id\", \"obs_date\", \"hba1c\", \"metformin_start\"]]\n", " .rename(columns={\"obs_date\": \"poor_ctrl_date\"})\n", ")\n", "\n", @@ -467,34 +481,31 @@ ], "source": [ "sl_claims = v_drug_claims[\n", - " v_drug_claims[\"patient_id\"].isin(poor_ctrl[\"patient_id\"]) &\n", - " v_drug_claims[\"ndcnum\"].isin(SECOND_LINE)\n", + " v_drug_claims[\"patient_id\"].isin(poor_ctrl[\"patient_id\"]) & v_drug_claims[\"ndcnum\"].isin(SECOND_LINE)\n", "].copy()\n", "sl_claims[\"drug_class\"] = sl_claims[\"ndcnum\"].map(DRUG_CLASS)\n", "\n", "# Merge with poor-control date\n", - "sl_claims = sl_claims.merge(\n", - " poor_ctrl[[\"patient_id\", \"poor_ctrl_date\"]], on=\"patient_id\", how=\"inner\"\n", - ")\n", + "sl_claims = sl_claims.merge(poor_ctrl[[\"patient_id\", \"poor_ctrl_date\"]], on=\"patient_id\", how=\"inner\")\n", "\n", "# Filter: after poor-control date and within 365 days\n", "sl_claims[\"days_after_ctrl\"] = (sl_claims[\"svcdate\"] - sl_claims[\"poor_ctrl_date\"]).dt.days\n", - "sl_claims = sl_claims[\n", - " (sl_claims[\"days_after_ctrl\"] >= 0) &\n", - " (sl_claims[\"days_after_ctrl\"] <= 365)\n", - "].sort_values([\"patient_id\", \"svcdate\"]).reset_index(drop=True)\n", + "sl_claims = (\n", + " sl_claims[(sl_claims[\"days_after_ctrl\"] >= 0) & (sl_claims[\"days_after_ctrl\"] <= 365)]\n", + " .sort_values([\"patient_id\", \"svcdate\"])\n", + " .reset_index(drop=True)\n", + ")\n", "\n", "# Keep first claim per patient (new initiation)\n", "sl_claims[\"prev_sl_date\"] = sl_claims.groupby(\"patient_id\")[\"svcdate\"].shift(1)\n", - "sl_claims[\"sl_gap\"] = (sl_claims[\"svcdate\"] - sl_claims[\"prev_sl_date\"]).dt.days\n", - "sl_claims[\"is_new\"] = sl_claims[\"prev_sl_date\"].isna() | (sl_claims[\"sl_gap\"] > 180)\n", + "sl_claims[\"sl_gap\"] = (sl_claims[\"svcdate\"] - sl_claims[\"prev_sl_date\"]).dt.days\n", + "sl_claims[\"is_new\"] = sl_claims[\"prev_sl_date\"].isna() | (sl_claims[\"sl_gap\"] > 180)\n", "\n", "cohort = (\n", " sl_claims[sl_claims[\"is_new\"]]\n", " .groupby(\"patient_id\")\n", " .first()\n", - " .reset_index()\n", - " [[\"patient_id\", \"svcdate\", \"drug_class\", \"poor_ctrl_date\"]]\n", + " .reset_index()[[\"patient_id\", \"svcdate\", \"drug_class\", \"poor_ctrl_date\"]]\n", " .rename(columns={\"svcdate\": \"second_line_start\"})\n", ")\n", "\n", @@ -531,12 +542,13 @@ } ], "source": [ - "AMI_ICD9_PFX = [\"410\"]\n", - "AMI_ICD10_PFX = [\"I21\", \"I22\"]\n", + "AMI_ICD9_PFX = [\"410\"]\n", + "AMI_ICD10_PFX = [\"I21\", \"I22\"]\n", "STROKE_ICD9_PFX = [\"160\", \"161\", \"162\", \"163\", \"430\", \"431\", \"432\", \"433\", \"434\", \"435\"]\n", - "STROKE_ICD10_PFX= [\"I60\", \"I61\", \"I62\", \"I63\", \"I64\"]\n", - "CHF_ICD9 = [\"4280\", \"4281\", \"4282\", \"4283\", \"4284\", \"4289\"]\n", - "CHF_ICD10_PFX = [\"I50\"]\n", + "STROKE_ICD10_PFX = [\"I60\", \"I61\", \"I62\", \"I63\", \"I64\"]\n", + "CHF_ICD9 = [\"4280\", \"4281\", \"4282\", \"4283\", \"4284\", \"4289\"]\n", + "CHF_ICD10_PFX = [\"I50\"]\n", + "\n", "\n", "def flag_outcome(dx_series, icd9_pfx=None, icd10_pfx=None, icd9_exact=None):\n", " m = pd.Series(False, index=dx_series.index)\n", @@ -550,21 +562,21 @@ " m |= dx_series.isin(icd9_exact)\n", " return m\n", "\n", + "\n", "inp = v_inpatient_services.copy()\n", "\n", "for col in [\"dx1\", \"dx2\"]:\n", " if col not in inp.columns:\n", " inp[col] = pd.NA\n", "\n", - "inp[\"ami\"] = flag_outcome(inp[\"dx1\"], icd9_pfx=AMI_ICD9_PFX, icd10_pfx=AMI_ICD10_PFX)\n", + "inp[\"ami\"] = flag_outcome(inp[\"dx1\"], icd9_pfx=AMI_ICD9_PFX, icd10_pfx=AMI_ICD10_PFX)\n", "inp[\"stroke\"] = flag_outcome(inp[\"dx1\"], icd9_pfx=STROKE_ICD9_PFX, icd10_pfx=STROKE_ICD10_PFX)\n", - "inp[\"chf\"] = flag_outcome(inp[\"dx1\"], icd9_exact=CHF_ICD9, icd10_pfx=CHF_ICD10_PFX)\n", - "inp[\"mace\"] = inp[\"ami\"] | inp[\"stroke\"] | inp[\"chf\"]\n", + "inp[\"chf\"] = flag_outcome(inp[\"dx1\"], icd9_exact=CHF_ICD9, icd10_pfx=CHF_ICD10_PFX)\n", + "inp[\"mace\"] = inp[\"ami\"] | inp[\"stroke\"] | inp[\"chf\"]\n", "\n", "# Merge with cohort and keep events after second-line start\n", - "inp_cohort = (\n", - " inp[inp[\"mace\"] & inp[\"patient_id\"].isin(cohort[\"patient_id\"])]\n", - " .merge(cohort[[\"patient_id\", \"second_line_start\"]], on=\"patient_id\", how=\"inner\")\n", + "inp_cohort = inp[inp[\"mace\"] & inp[\"patient_id\"].isin(cohort[\"patient_id\"])].merge(\n", + " cohort[[\"patient_id\", \"second_line_start\"]], on=\"patient_id\", how=\"inner\"\n", ")\n", "inp_cohort = inp_cohort[inp_cohort[\"svcdate\"] > inp_cohort[\"second_line_start\"]]\n", "\n", @@ -651,6 +663,7 @@ "source": [ "enrind_cols = [f\"enrind{m}\" for m in range(1, 13)]\n", "\n", + "\n", "def followup_months(pid, start_date, enrollment_df, enrind_cols):\n", " \"\"\"Return months of continuous enrollment after start_date.\"\"\"\n", " rows = enrollment_df[enrollment_df[\"patient_id\"] == pid].sort_values(\"year\")\n", @@ -665,6 +678,7 @@ " count += 1\n", " return count\n", "\n", + "\n", "cohort_follow = cohort.copy()\n", "cohort_follow[\"followup_months\"] = cohort_follow.apply(\n", " lambda r: followup_months(r[\"patient_id\"], r[\"second_line_start\"], v_annual, enrind_cols),\n", @@ -730,9 +744,7 @@ "diag_raw = v_outpatient[[\"patient_id\", \"svcdate\", \"dx1\", \"dxver\"]].copy()\n", "diag_raw = diag_raw[diag_raw[\"patient_id\"].isin(cohort_ids)]\n", "diag_raw = diag_raw.rename(columns={\"svcdate\": \"eventdate\", \"dx1\": \"dx\"})\n", - "diag_raw[\"dxver\"] = diag_raw[\"dxver\"].fillna(\n", - " diag_raw[\"dx\"].apply(lambda d: \"9\" if d and d[0].isdigit() else \"0\")\n", - ")\n", + "diag_raw[\"dxver\"] = diag_raw[\"dxver\"].fillna(diag_raw[\"dx\"].apply(lambda d: \"9\" if d and d[0].isdigit() else \"0\"))\n", "\n", "# therapy canonical table from drug claims\n", "therapy_raw = (\n", @@ -745,9 +757,19 @@ " start_date=pd.NaT,\n", " end_date=pd.NaT,\n", " refill=pd.array([pd.NA] * len(v_drug_claims[v_drug_claims[\"patient_id\"].isin(cohort_ids)]), dtype=\"Int64\"),\n", - " )\n", - " [[\"patient_id\", \"fill_date\", \"ingredient\", \"ndc11\", \"rxcui\",\n", - " \"prescription_date\", \"start_date\", \"end_date\", \"refill\"]]\n", + " )[\n", + " [\n", + " \"patient_id\",\n", + " \"fill_date\",\n", + " \"ingredient\",\n", + " \"ndc11\",\n", + " \"rxcui\",\n", + " \"prescription_date\",\n", + " \"start_date\",\n", + " \"end_date\",\n", + " \"refill\",\n", + " ]\n", + " ]\n", ")\n", "\n", "edata = to_ehrdata(\n", @@ -779,15 +801,14 @@ ], "source": [ "# Attach cohort metadata to obs\n", - "cohort_meta = (\n", - " cohort_follow[[\"patient_id\", \"drug_class\", \"second_line_start\", \"followup_months\"]]\n", - " .set_index(\"patient_id\")\n", + "cohort_meta = cohort_follow[[\"patient_id\", \"drug_class\", \"second_line_start\", \"followup_months\"]].set_index(\n", + " \"patient_id\"\n", ")\n", "# Align to edata.obs index\n", "common = edata.obs_names.intersection(cohort_meta.index)\n", - "edata.obs.loc[common, \"drug_class\"] = cohort_meta.loc[common, \"drug_class\"]\n", + "edata.obs.loc[common, \"drug_class\"] = cohort_meta.loc[common, \"drug_class\"]\n", "edata.obs.loc[common, \"second_line_start\"] = cohort_meta.loc[common, \"second_line_start\"]\n", - "edata.obs.loc[common, \"followup_months\"] = cohort_meta.loc[common, \"followup_months\"]\n", + "edata.obs.loc[common, \"followup_months\"] = cohort_meta.loc[common, \"followup_months\"]\n", "\n", "print(\"obs columns:\", edata.obs.columns.tolist())\n", "print()\n", diff --git a/docs/tutorials/source_loinc_mapping.ipynb b/docs/tutorials/source_loinc_mapping.ipynb index 032704e9..c01706f1 100644 --- a/docs/tutorials/source_loinc_mapping.ipynb +++ b/docs/tutorials/source_loinc_mapping.ipynb @@ -27,8 +27,7 @@ "import numpy as np\n", "import pandas as pd\n", "\n", - "from ehrdata.io.source.adapters import lced\n", - "from ehrdata.io.source.vocab import loinc as loinc_vocab" + "from ehrdata.io.source.adapters import lced" ] }, { @@ -151,40 +150,54 @@ "source": [ "# Representative LOINC codes used in T2D / cardiometabolic research\n", "# In practice, load from loinc_map.csv (subset of the full LOINC table)\n", - "loinc_reference = pd.DataFrame({\n", - " \"loinc_code\": [\n", - " \"4548-4\", \"17856-6\", \"59261-8\", # HbA1c\n", - " \"29463-7\", \"3141-9\", \"75292-3\", # Weight\n", - " \"8302-2\", \"3137-7\", \"8308-9\", # Height\n", - " \"39156-5\", \"59574-4\", \"89270-3\", # BMI\n", - " \"2345-7\", \"14749-6\", # Glucose\n", - " \"2160-0\", \"38483-4\", # Creatinine\n", - " \"4544-3\", \"20570-8\", # Haematocrit\n", - " \"718-7\", \"26515-7\", # Haemoglobin, Platelets\n", - " ],\n", - " \"long_common_name\": [\n", - " \"Hemoglobin A1c/Hemoglobin.total in Blood\",\n", - " \"Hemoglobin A1c/Hemoglobin.total in Blood by HPLC\",\n", - " \"Hemoglobin A1c/Hemoglobin.total in Blood by Immunoassay\",\n", - " \"Body weight\",\n", - " \"Body weight Measured\",\n", - " \"Body weight - Reported\",\n", - " \"Body height\",\n", - " \"Body height Measured\",\n", - " \"Body height - standing\",\n", - " \"Body mass index (BMI) [Ratio]\",\n", - " \"Body mass index (BMI) [Percentile]\",\n", - " \"Body mass index (BMI) [Ratio] Reported\",\n", - " \"Glucose [Mass/volume] in Serum or Plasma\",\n", - " \"Glucose [Moles/volume] in Serum or Plasma\",\n", - " \"Creatinine [Mass/volume] in Serum or Plasma\",\n", - " \"Creatinine [Mass/volume] in Blood\",\n", - " \"Hematocrit [Volume Fraction] of Blood by Automated count\",\n", - " \"Hematocrit [Volume Fraction] of Blood\",\n", - " \"Hemoglobin [Mass/volume] in Blood\",\n", - " \"Platelets [#/volume] in Blood by Automated count\",\n", - " ],\n", - "})\n", + "loinc_reference = pd.DataFrame(\n", + " {\n", + " \"loinc_code\": [\n", + " \"4548-4\",\n", + " \"17856-6\",\n", + " \"59261-8\", # HbA1c\n", + " \"29463-7\",\n", + " \"3141-9\",\n", + " \"75292-3\", # Weight\n", + " \"8302-2\",\n", + " \"3137-7\",\n", + " \"8308-9\", # Height\n", + " \"39156-5\",\n", + " \"59574-4\",\n", + " \"89270-3\", # BMI\n", + " \"2345-7\",\n", + " \"14749-6\", # Glucose\n", + " \"2160-0\",\n", + " \"38483-4\", # Creatinine\n", + " \"4544-3\",\n", + " \"20570-8\", # Haematocrit\n", + " \"718-7\",\n", + " \"26515-7\", # Haemoglobin, Platelets\n", + " ],\n", + " \"long_common_name\": [\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood\",\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood by HPLC\",\n", + " \"Hemoglobin A1c/Hemoglobin.total in Blood by Immunoassay\",\n", + " \"Body weight\",\n", + " \"Body weight Measured\",\n", + " \"Body weight - Reported\",\n", + " \"Body height\",\n", + " \"Body height Measured\",\n", + " \"Body height - standing\",\n", + " \"Body mass index (BMI) [Ratio]\",\n", + " \"Body mass index (BMI) [Percentile]\",\n", + " \"Body mass index (BMI) [Ratio] Reported\",\n", + " \"Glucose [Mass/volume] in Serum or Plasma\",\n", + " \"Glucose [Moles/volume] in Serum or Plasma\",\n", + " \"Creatinine [Mass/volume] in Serum or Plasma\",\n", + " \"Creatinine [Mass/volume] in Blood\",\n", + " \"Hematocrit [Volume Fraction] of Blood by Automated count\",\n", + " \"Hematocrit [Volume Fraction] of Blood\",\n", + " \"Hemoglobin [Mass/volume] in Blood\",\n", + " \"Platelets [#/volume] in Blood by Automated count\",\n", + " ],\n", + " }\n", + ")\n", "\n", "print(f\"Reference LOINC codes: {len(loinc_reference)}\")\n", "loinc_reference.head(8)" @@ -301,16 +314,16 @@ ], "source": [ "# Canonical concept code lists (union of variants)\n", - "HBA1C_LOINCS = [\"4548-4\", \"17856-6\", \"59261-8\"]\n", + "HBA1C_LOINCS = [\"4548-4\", \"17856-6\", \"59261-8\"]\n", "WEIGHT_LOINCS = [\"29463-7\", \"3141-9\", \"75292-3\"]\n", "HEIGHT_LOINCS = [\"8302-2\", \"3137-7\", \"8308-9\"]\n", - "BMI_LOINCS = [\"39156-5\", \"59574-4\", \"89270-3\"]\n", + "BMI_LOINCS = [\"39156-5\", \"59574-4\", \"89270-3\"]\n", "\n", "all_concept_loincs = {\n", - " \"hba1c\": HBA1C_LOINCS,\n", + " \"hba1c\": HBA1C_LOINCS,\n", " \"weight\": WEIGHT_LOINCS,\n", " \"height\": HEIGHT_LOINCS,\n", - " \"bmi\": BMI_LOINCS,\n", + " \"bmi\": BMI_LOINCS,\n", "}\n", "for concept, codes in all_concept_loincs.items():\n", " print(f\"{concept:8s}: {codes}\")" @@ -348,27 +361,31 @@ "patient_pool = [f\"P{i:04d}\" for i in range(1, 11)]\n", "\n", "# Explorys v_observation (EMR)\n", - "v_observation = pd.DataFrame({\n", - " \"patient_id\": np.random.choice(patient_pool, n),\n", - " \"observation_date\": pd.date_range(\"2014-01-01\", periods=n, freq=\"14D\"),\n", - " \"loinc_test_id\": np.random.choice(\n", - " HBA1C_LOINCS + WEIGHT_LOINCS + HEIGHT_LOINCS + BMI_LOINCS + [\"2345-7\", \"2160-0\"],\n", - " n,\n", - " ),\n", - " \"std_value\": np.round(np.random.uniform(4.0, 12.0, n), 1),\n", - " \"std_uom\": np.random.choice([\"%\", \"kg\", \"cm\", \"kg/m2\", \"mg/dL\"], n),\n", - "})\n", + "v_observation = pd.DataFrame(\n", + " {\n", + " \"patient_id\": np.random.choice(patient_pool, n),\n", + " \"observation_date\": pd.date_range(\"2014-01-01\", periods=n, freq=\"14D\"),\n", + " \"loinc_test_id\": np.random.choice(\n", + " HBA1C_LOINCS + WEIGHT_LOINCS + HEIGHT_LOINCS + BMI_LOINCS + [\"2345-7\", \"2160-0\"],\n", + " n,\n", + " ),\n", + " \"std_value\": np.round(np.random.uniform(4.0, 12.0, n), 1),\n", + " \"std_uom\": np.random.choice([\"%\", \"kg\", \"cm\", \"kg/m2\", \"mg/dL\"], n),\n", + " }\n", + ")\n", "\n", "# MarketScan v_lab_results (claims)\n", "n2 = 20\n", - "v_lab_results = pd.DataFrame({\n", - " \"patient_id\": np.random.choice(patient_pool, n2),\n", - " \"svcdate\": pd.date_range(\"2014-03-01\", periods=n2, freq=\"21D\"),\n", - " \"loinccd\": np.random.choice(HBA1C_LOINCS + WEIGHT_LOINCS + [\"2345-7\"], n2),\n", - " \"result\": np.round(np.random.uniform(4.0, 12.0, n2), 1),\n", - " \"resunit\": np.random.choice([\"%\", \"kg\", \"mg/dL\"], n2),\n", - " \"resltcat\": np.random.choice([None, \"Normal\", \"High\", \"Low\"], n2),\n", - "})\n", + "v_lab_results = pd.DataFrame(\n", + " {\n", + " \"patient_id\": np.random.choice(patient_pool, n2),\n", + " \"svcdate\": pd.date_range(\"2014-03-01\", periods=n2, freq=\"21D\"),\n", + " \"loinccd\": np.random.choice(HBA1C_LOINCS + WEIGHT_LOINCS + [\"2345-7\"], n2),\n", + " \"result\": np.round(np.random.uniform(4.0, 12.0, n2), 1),\n", + " \"resunit\": np.random.choice([\"%\", \"kg\", \"mg/dL\"], n2),\n", + " \"resltcat\": np.random.choice([None, \"Normal\", \"High\", \"Low\"], n2),\n", + " }\n", + ")\n", "\n", "print(f\"v_observation rows : {len(v_observation)}\")\n", "print(f\"v_lab_results rows : {len(v_lab_results)}\")" @@ -669,8 +686,7 @@ ], "source": [ "a1c = (\n", - " hba1c_labs\n", - " .dropna(subset=[\"value\"])\n", + " hba1c_labs.dropna(subset=[\"value\"])\n", " .query(\"value >= 7\")\n", " .sort_values([\"patient_id\", \"eventdate\"])\n", " .reset_index(drop=True)\n", @@ -749,12 +765,16 @@ " .gt(0) # boolean: has any measurement\n", " )\n", "\n", - "coverage = pd.concat([\n", - " concept_flag(labs, HBA1C_LOINCS, \"hba1c\"),\n", - " concept_flag(labs, WEIGHT_LOINCS, \"weight\"),\n", - " concept_flag(labs, HEIGHT_LOINCS, \"height\"),\n", - " concept_flag(labs, BMI_LOINCS, \"bmi\"),\n", - "], axis=1).fillna(False)\n", + "\n", + "coverage = pd.concat(\n", + " [\n", + " concept_flag(labs, HBA1C_LOINCS, \"hba1c\"),\n", + " concept_flag(labs, WEIGHT_LOINCS, \"weight\"),\n", + " concept_flag(labs, HEIGHT_LOINCS, \"height\"),\n", + " concept_flag(labs, BMI_LOINCS, \"bmi\"),\n", + " ],\n", + " axis=1,\n", + ").fillna(False)\n", "\n", "print(\"Concept coverage per patient:\")\n", "print(coverage)\n", diff --git a/src/ehrdata/io/source/adapters/cprd.py b/src/ehrdata/io/source/adapters/cprd.py index c641db39..1c3dbd76 100644 --- a/src/ehrdata/io/source/adapters/cprd.py +++ b/src/ehrdata/io/source/adapters/cprd.py @@ -149,8 +149,9 @@ def build_therapy( out["rxcui"] = pd.array([None] * len(out), dtype=object) out["ndc11"] = pd.array([None] * len(out), dtype=object) - out = out[[_PID, "prescription_date", "start_date", "fill_date", "end_date", - "refill", "rxcui", "ndc11", "ingredient"]] + out = out[ + [_PID, "prescription_date", "start_date", "fill_date", "end_date", "refill", "rxcui", "ndc11", "ingredient"] + ] return deduplicate(out) @@ -209,12 +210,14 @@ def build_labtest( if entity_enttypes is not None: out = out[out["enttype"].isin(entity_enttypes)] - out = out.rename(columns={ - "patid": _PID, - "data2": "value", - "data3": "unit", - "data4": "valuecat", - }) + out = out.rename( + columns={ + "patid": _PID, + "data2": "value", + "data3": "unit", + "data4": "valuecat", + } + ) out[_PID] = coerce_patient_id(out[_PID]) out["eventdate"] = coerce_date(out["eventdate"], formats=[_CPRD_DATE_FMT]) out["loinc"] = pd.array([None] * len(out), dtype=object) diff --git a/src/ehrdata/io/source/adapters/lced.py b/src/ehrdata/io/source/adapters/lced.py index 0ca90d5e..5fdc11a3 100644 --- a/src/ehrdata/io/source/adapters/lced.py +++ b/src/ehrdata/io/source/adapters/lced.py @@ -83,15 +83,11 @@ def build_diagnosis( :data:`~ehrdata.io.source.schema.DIAGNOSIS`. """ parts = [ - _unnest_dx(facility_header, date_col="svcdate", - dx_cols=[f"dx{i}" for i in range(1, 10)]), - _unnest_dx(inpatient_admissions, date_col="admdate", - dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), - _unnest_dx(inpatient_services, date_col="svcdate", - dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), - _unnest_dx(lab_results, date_col="svcdate", dx_cols=["dx1"]), - _unnest_dx(outpatient_services, date_col="svcdate", - dx_cols=[f"dx{i}" for i in range(1, 5)]), + _unnest_dx(facility_header, date_col="svcdate", dx_cols=[f"dx{i}" for i in range(1, 10)]), + _unnest_dx(inpatient_admissions, date_col="admdate", dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), + _unnest_dx(inpatient_services, date_col="svcdate", dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), + _unnest_dx(lab_results, date_col="svcdate", dx_cols=["dx1"]), + _unnest_dx(outpatient_services, date_col="svcdate", dx_cols=[f"dx{i}" for i in range(1, 5)]), ] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) @@ -106,7 +102,7 @@ def _unnest_dx(src: pd.DataFrame, *, date_col: str, dx_cols: list[str]) -> pd.Da """Select and unnest diagnosis codes from one LCED source table.""" present_dx = [c for c in dx_cols if c in src.columns] dxver_cols = ["dxver"] if "dxver" in src.columns else [] - tmp = src[[_PID, date_col] + dxver_cols + present_dx].copy() + tmp = src[[_PID, date_col, *dxver_cols, *present_dx]].copy() tmp = tmp.rename(columns={date_col: "eventdate"}) if "dxver" not in tmp.columns: tmp["dxver"] = None @@ -152,8 +148,19 @@ def build_therapy( claims_part = _build_claims_part(v_outpatient_drug_claims, ndc_map=ndc_map) df = union_tables([drug_part, claims_part]) df = deduplicate(df) - return df[["patient_id", "prescription_date", "start_date", "fill_date", - "end_date", "refill", "rxcui", "ndc11", "ingredient"]] + return df[ + [ + "patient_id", + "prescription_date", + "start_date", + "fill_date", + "end_date", + "refill", + "rxcui", + "ndc11", + "ingredient", + ] + ] def _build_drug_part(src: pd.DataFrame, *, rxcui_map: pd.DataFrame | None) -> pd.DataFrame: @@ -169,6 +176,7 @@ def _build_drug_part(src: pd.DataFrame, *, rxcui_map: pd.DataFrame | None) -> pd df["ndc11"] = None if rxcui_map is not None: from ehrdata.io.source.vocab.rxnorm import join_ingredient_by_rxcui + df = join_ingredient_by_rxcui(df, rxcui_map) else: df["ingredient"] = None @@ -189,6 +197,7 @@ def _build_claims_part(src: pd.DataFrame, *, ndc_map: pd.DataFrame | None) -> pd df["ndc11"] = src["ndcnum"].astype(str).str.strip().str.zfill(11) if ndc_map is not None: from ehrdata.io.source.vocab.ndc import join_ingredient_by_ndc + df = join_ingredient_by_ndc(df, ndc_map) else: df["ingredient"] = None @@ -251,8 +260,10 @@ def _build_lab_results_part(src: pd.DataFrame) -> pd.DataFrame: df = pd.DataFrame() df[_PID] = src[_PID] df["eventdate"] = src.get("svcdate") - df["value"] = src.get("result", pd.Series(dtype=object, index=src.index)).astype(str).where( - src.get("result", pd.Series(dtype=object, index=src.index)).notna(), None + df["value"] = ( + src.get("result", pd.Series(dtype=object, index=src.index)) + .astype(str) + .where(src.get("result", pd.Series(dtype=object, index=src.index)).notna(), None) ) df["valuecat"] = src.get("resltcat", pd.Series(dtype=object, index=src.index)) df["unit"] = src.get("resunit", pd.Series(dtype=object, index=src.index)) @@ -299,16 +310,18 @@ def build_procedure( :data:`~ehrdata.io.source.schema.PROCEDURE`. """ parts = [ - _unnest_proc(facility_header, date_col="svcdate", - proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None), - _unnest_proc(inpatient_admissions, date_col="admdate", - proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], proctype_col=None), - _unnest_proc(inpatient_services, date_col="svcdate", - proc_cols=["pdx", "proc1"], proctype_col="proctyp"), - _unnest_proc(lab_results, date_col="svcdate", - proc_cols=["proc1"], proctype_col="proctyp"), - _unnest_proc(outpatient_services, date_col="svcdate", - proc_cols=["proc1"], proctype_col="proctyp"), + _unnest_proc( + facility_header, date_col="svcdate", proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None + ), + _unnest_proc( + inpatient_admissions, + date_col="admdate", + proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], + proctype_col=None, + ), + _unnest_proc(inpatient_services, date_col="svcdate", proc_cols=["pdx", "proc1"], proctype_col="proctyp"), + _unnest_proc(lab_results, date_col="svcdate", proc_cols=["proc1"], proctype_col="proctyp"), + _unnest_proc(outpatient_services, date_col="svcdate", proc_cols=["proc1"], proctype_col="proctyp"), ] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) @@ -327,7 +340,7 @@ def _unnest_proc( ) -> pd.DataFrame: """Select and unnest procedure codes from one LCED source table.""" present_proc = [c for c in proc_cols if c in src.columns] - tmp = src[[_PID, date_col] + present_proc].copy() + tmp = src[[_PID, date_col, *present_proc]].copy() tmp = tmp.rename(columns={date_col: "eventdate"}) if proctype_col and proctype_col in src.columns: tmp["proctype"] = src[proctype_col].values @@ -373,8 +386,7 @@ def build_habit( df["encounter_date"] = coerce_date(df["encounter_date"]) df = df.drop(columns="encounter_join_id") df = deduplicate(df) - df = df.sort_values([_PID, "encounter_date", "mapped_question_answer"], - na_position="last").reset_index(drop=True) + df = df.sort_values([_PID, "encounter_date", "mapped_question_answer"], na_position="last").reset_index(drop=True) return df[[_PID, "encounter_date", "mapped_question_answer"]] @@ -401,8 +413,7 @@ def build_patinfo(*source_tables: pd.DataFrame) -> pd.DataFrame: Canonical patinfo DataFrame with columns ``patient_id``, ``dobyr``, ``sex``. """ - parts = [t[[c for c in ["patient_id", "dobyr", "sex"] if c in t.columns]].copy() - for t in source_tables] + parts = [t[[c for c in ["patient_id", "dobyr", "sex"] if c in t.columns]].copy() for t in source_tables] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) df["dobyr"] = pd.to_numeric(df["dobyr"], errors="coerce").astype("Int64") @@ -436,8 +447,10 @@ def build_insurance( :data:`~ehrdata.io.source.schema.INSURANCE`. """ _COLS = ["patient_id", "svcdate", "cob", "coins", "copay"] - parts = [src[[c for c in _COLS if c in src.columns]].copy() - for src in (facility_header, inpatient_services, outpatient_drug_claims, outpatient_services)] + parts = [ + src[[c for c in _COLS if c in src.columns]].copy() + for src in (facility_header, inpatient_services, outpatient_drug_claims, outpatient_services) + ] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) df["svcdate"] = coerce_date(df["svcdate"]) diff --git a/src/ehrdata/io/source/adapters/marketscan.py b/src/ehrdata/io/source/adapters/marketscan.py index e6e4a631..f1099013 100644 --- a/src/ehrdata/io/source/adapters/marketscan.py +++ b/src/ehrdata/io/source/adapters/marketscan.py @@ -43,12 +43,20 @@ # Patinfo columns that exist in MarketScan beyond the canonical three _PATINFO_EXTRA_COLS = [ - "efamid", "year", "region", "msa", "wgtkey", - "eeclass", "eestatu", "egeoloc", "emprel", "indstry", + "efamid", + "year", + "region", + "msa", + "wgtkey", + "eeclass", + "eestatu", + "egeoloc", + "emprel", + "indstry", ] # All patinfo columns as they appear in MarketScan source tables -_PATINFO_SRC_COLS = [_ENROLID, "dobyr", "sex"] + _PATINFO_EXTRA_COLS +_PATINFO_SRC_COLS = [_ENROLID, "dobyr", "sex", *_PATINFO_EXTRA_COLS] # --------------------------------------------------------------------------- @@ -84,14 +92,10 @@ def build_diagnosis( :data:`~ehrdata.io.source.schema.DIAGNOSIS`. """ parts = [ - _unnest_dx(facility_header, date_col="svcdate", - dx_cols=[f"dx{i}" for i in range(1, 10)]), - _unnest_dx(inpatient_admissions, date_col="admdate", - dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), - _unnest_dx(inpatient_services, date_col="svcdate", - dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), - _unnest_dx(outpatient_services, date_col="svcdate", - dx_cols=[f"dx{i}" for i in range(1, 5)]), + _unnest_dx(facility_header, date_col="svcdate", dx_cols=[f"dx{i}" for i in range(1, 10)]), + _unnest_dx(inpatient_admissions, date_col="admdate", dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 16)]), + _unnest_dx(inpatient_services, date_col="svcdate", dx_cols=["pdx"] + [f"dx{i}" for i in range(1, 5)]), + _unnest_dx(outpatient_services, date_col="svcdate", dx_cols=[f"dx{i}" for i in range(1, 5)]), ] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) @@ -106,7 +110,7 @@ def _unnest_dx(src: pd.DataFrame, *, date_col: str, dx_cols: list[str]) -> pd.Da """Select, rename, and unnest diagnosis codes from one source table.""" present_dx = [c for c in dx_cols if c in src.columns] dxver_col = ["dxver"] if "dxver" in src.columns else [] - tmp = src[[_ENROLID, date_col] + dxver_col + present_dx].copy() + tmp = src[[_ENROLID, date_col, *dxver_col, *present_dx]].copy() tmp = tmp.rename(columns={_ENROLID: _PID, date_col: "eventdate"}) if "dxver" not in tmp.columns: tmp["dxver"] = None @@ -159,13 +163,25 @@ def build_therapy( if ndc_map is not None: from ehrdata.io.source.vocab.ndc import join_ingredient_by_ndc + df = join_ingredient_by_ndc(df, ndc_map) else: df["ingredient"] = None df = deduplicate(df) - return df[["patient_id", "prescription_date", "start_date", "fill_date", - "end_date", "refill", "rxcui", "ndc11", "ingredient"]] + return df[ + [ + "patient_id", + "prescription_date", + "start_date", + "fill_date", + "end_date", + "refill", + "rxcui", + "ndc11", + "ingredient", + ] + ] # --------------------------------------------------------------------------- @@ -202,14 +218,17 @@ def build_procedure( :data:`~ehrdata.io.source.schema.PROCEDURE`. """ parts = [ - _unnest_proc(facility_header, date_col="svcdate", - proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None), - _unnest_proc(inpatient_admissions, date_col="admdate", - proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], proctype_col=None), - _unnest_proc(inpatient_services, date_col="svcdate", - proc_cols=["pdx", "proc1"], proctype_col="proctyp"), - _unnest_proc(outpatient_services, date_col="svcdate", - proc_cols=["proc1"], proctype_col="proctyp"), + _unnest_proc( + facility_header, date_col="svcdate", proc_cols=[f"proc{i}" for i in range(1, 7)], proctype_col=None + ), + _unnest_proc( + inpatient_admissions, + date_col="admdate", + proc_cols=["pproc"] + [f"proc{i}" for i in range(1, 16)], + proctype_col=None, + ), + _unnest_proc(inpatient_services, date_col="svcdate", proc_cols=["pdx", "proc1"], proctype_col="proctyp"), + _unnest_proc(outpatient_services, date_col="svcdate", proc_cols=["proc1"], proctype_col="proctyp"), ] df = union_tables(parts) df[_PID] = coerce_patient_id(df[_PID]) @@ -228,7 +247,7 @@ def _unnest_proc( ) -> pd.DataFrame: """Select, rename, and unnest procedure codes from one source table.""" present_proc = [c for c in proc_cols if c in src.columns] - tmp = src[[_ENROLID, date_col] + present_proc].copy() + tmp = src[[_ENROLID, date_col, *present_proc]].copy() tmp = tmp.rename(columns={_ENROLID: _PID, date_col: "eventdate"}) if proctype_col and proctype_col in src.columns: tmp["proctype"] = src[proctype_col].values @@ -264,10 +283,7 @@ def build_patinfo(*source_tables: pd.DataFrame) -> pd.DataFrame: present. """ _REQUIRED = [_ENROLID, "dobyr", "sex"] - available_extra = [ - c for c in _PATINFO_EXTRA_COLS - if all(c in t.columns for t in source_tables) - ] + available_extra = [c for c in _PATINFO_EXTRA_COLS if all(c in t.columns for t in source_tables)] keep_cols = _REQUIRED + available_extra parts = [t[[c for c in keep_cols if c in t.columns]].copy() for t in source_tables] diff --git a/src/ehrdata/io/source/extract.py b/src/ehrdata/io/source/extract.py index a713f0cd..12a121e8 100644 --- a/src/ehrdata/io/source/extract.py +++ b/src/ehrdata/io/source/extract.py @@ -13,13 +13,12 @@ from __future__ import annotations import zipfile -from pathlib import Path from typing import TYPE_CHECKING import pandas as pd if TYPE_CHECKING: - pass + from pathlib import Path def union_tables(dfs: list[pd.DataFrame]) -> pd.DataFrame: @@ -39,7 +38,8 @@ def union_tables(dfs: list[pd.DataFrame]) -> pd.DataFrame: ValueError: If *dfs* is empty. """ if not dfs: - raise ValueError("dfs must contain at least one DataFrame") + msg = "dfs must contain at least one DataFrame" + raise ValueError(msg) return pd.concat(dfs, ignore_index=True).drop_duplicates().reset_index(drop=True) @@ -77,12 +77,7 @@ def unnest_codes( value_vars=code_cols, value_name=value_name, ) - return ( - long.dropna(subset=[value_name]) - .drop(columns="variable") - .drop_duplicates() - .reset_index(drop=True) - ) + return long.dropna(subset=[value_name]).drop(columns="variable").drop_duplicates().reset_index(drop=True) def read_zipped_tsv( @@ -92,7 +87,7 @@ def read_zipped_tsv( usecols: list[str] | None = None, **kwargs, ) -> pd.DataFrame: - """Read a single TSV member from a zip archive. + r"""Read a single TSV member from a zip archive. Mirrors ``fread(cmd=paste("unzip -p", file), sep="\\t")`` used in the CPRD R ETL to stream individual extract files without unzipping to disk. @@ -106,9 +101,8 @@ def read_zipped_tsv( Returns: DataFrame with the member's contents. """ - with zipfile.ZipFile(zip_path) as zf: - with zf.open(member) as fh: - return pd.read_csv(fh, sep="\t", usecols=usecols, **kwargs) + with zipfile.ZipFile(zip_path) as zf, zf.open(member) as fh: + return pd.read_csv(fh, sep="\t", usecols=usecols, **kwargs) def read_zipped_tsvs( @@ -179,4 +173,4 @@ def read_csv_with_duckdb( try: return con.execute(sql).fetchdf() finally: - con.close() \ No newline at end of file + con.close() diff --git a/src/ehrdata/io/source/normalize.py b/src/ehrdata/io/source/normalize.py index add3197c..887c9bd5 100644 --- a/src/ehrdata/io/source/normalize.py +++ b/src/ehrdata/io/source/normalize.py @@ -11,7 +11,6 @@ import pandas as pd - # ICD-9 uses E-codes (external causes) and V-codes (supplemental factors). # When ``dxver`` is missing in a claims record, a leading E or V reliably # identifies ICD-9 coding — heuristic taken from the original LCED ETL. diff --git a/src/ehrdata/io/source/schema.py b/src/ehrdata/io/source/schema.py index d923a188..425f08f7 100644 --- a/src/ehrdata/io/source/schema.py +++ b/src/ehrdata/io/source/schema.py @@ -84,7 +84,7 @@ def validate(self, df: pd.DataFrame, *, strict: bool = False) -> list[str]: name="diagnosis", columns=( ColumnSpec("patient_id", "object", nullable=False), - ColumnSpec("dxver", "object"), # "0" = ICD-10, "9" = ICD-9, None = unknown + ColumnSpec("dxver", "object"), # "0" = ICD-10, "9" = ICD-9, None = unknown ColumnSpec("eventdate", "datetime64[ns]"), ColumnSpec("dx", "object", nullable=False), ), @@ -141,8 +141,8 @@ def validate(self, df: pd.DataFrame, *, strict: bool = False) -> list[str]: columns=( ColumnSpec("patient_id", "object", nullable=False), ColumnSpec("svcdate", "datetime64[ns]"), - ColumnSpec("cob", "float64"), # coordination of benefits amount - ColumnSpec("coins", "float64"), # coinsurance amount + ColumnSpec("cob", "float64"), # coordination of benefits amount + ColumnSpec("coins", "float64"), # coinsurance amount ColumnSpec("copay", "float64"), ), ) @@ -154,7 +154,7 @@ def validate(self, df: pd.DataFrame, *, strict: bool = False) -> list[str]: ColumnSpec("dtstart", "datetime64[ns]"), ColumnSpec("dtend", "datetime64[ns]"), ColumnSpec("plantyp", "object"), - ColumnSpec("rx", "object"), # pharmacy coverage flag + ColumnSpec("rx", "object"), # pharmacy coverage flag ColumnSpec("hlthplan", "object"), ), ) @@ -169,6 +169,5 @@ def validate(self, df: pd.DataFrame, *, strict: bool = False) -> list[str]: ) ALL_SCHEMAS: dict[str, TableSchema] = { - s.name: s - for s in (DIAGNOSIS, THERAPY, LABTEST, PROCEDURE, PATINFO, INSURANCE, PROVIDER, HABIT) -} \ No newline at end of file + s.name: s for s in (DIAGNOSIS, THERAPY, LABTEST, PROCEDURE, PATINFO, INSURANCE, PROVIDER, HABIT) +} diff --git a/src/ehrdata/io/source/to_ehrdata.py b/src/ehrdata/io/source/to_ehrdata.py index 792add2f..d7ab31f3 100644 --- a/src/ehrdata/io/source/to_ehrdata.py +++ b/src/ehrdata/io/source/to_ehrdata.py @@ -46,7 +46,7 @@ def to_ehrdata( labtest: pd.DataFrame | None = None, procedure: pd.DataFrame | None = None, source: str | None = None, -) -> "EHRData": +) -> EHRData: """Convert canonical source tables into an :class:`~ehrdata.EHRData` presence matrix. Args: @@ -76,7 +76,14 @@ def to_ehrdata( >>> import pandas as pd >>> import ehrdata as ed >>> patinfo = pd.DataFrame({"patient_id": ["P1", "P2"], "dobyr": [1960, 1975], "sex": ["M", "F"]}) - >>> diagnosis = pd.DataFrame({"patient_id": ["P1", "P1", "P2"], "dx": ["E11.9", "I10", "E11.9"], "dxver": [None, None, None], "eventdate": pd.NaT}) + >>> diagnosis = pd.DataFrame( + ... { + ... "patient_id": ["P1", "P1", "P2"], + ... "dx": ["E11.9", "I10", "E11.9"], + ... "dxver": [None, None, None], + ... "eventdate": pd.NaT, + ... } + ... ) >>> edata = ed.io.source.to_ehrdata(patinfo, diagnosis=diagnosis, source="example") >>> edata.obs_names.tolist() ['P1', 'P2'] @@ -176,8 +183,10 @@ def _append_pairs( if sub.empty: return frames.append( - pd.DataFrame({ - "patient_id": sub[pid_col].astype(str).values, - "concept": prefix + ":" + sub[code_col].astype(str).values, - }) + pd.DataFrame( + { + "patient_id": sub[pid_col].astype(str).values, + "concept": prefix + ":" + sub[code_col].astype(str).values, + } + ) ) diff --git a/src/ehrdata/io/source/vocab/icd.py b/src/ehrdata/io/source/vocab/icd.py index 090dd423..2dbda477 100644 --- a/src/ehrdata/io/source/vocab/icd.py +++ b/src/ehrdata/io/source/vocab/icd.py @@ -7,7 +7,10 @@ from __future__ import annotations -import pandas as pd +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pandas as pd # ICD-9 code series that begin with E (external causes) or V (supplemental # factors) are unambiguously ICD-9 when the version flag is absent in claims @@ -67,7 +70,8 @@ def load_gem_map(path: str) -> pd.DataFrame: # pragma: no cover Raises: NotImplementedError: Always — implementation pending external data. """ - raise NotImplementedError("GEM loader requires external CMS GEM file; not yet implemented.") + msg = "GEM loader requires external CMS GEM file; not yet implemented." + raise NotImplementedError(msg) def load_ccs_map(path: str) -> pd.DataFrame: # pragma: no cover @@ -83,4 +87,5 @@ def load_ccs_map(path: str) -> pd.DataFrame: # pragma: no cover Raises: NotImplementedError: Always — implementation pending external data. """ - raise NotImplementedError("CCS loader requires external AHRQ CCS file; not yet implemented.") + msg = "CCS loader requires external AHRQ CCS file; not yet implemented." + raise NotImplementedError(msg) diff --git a/src/ehrdata/io/source/vocab/loinc.py b/src/ehrdata/io/source/vocab/loinc.py index abe628ff..85fb0dcf 100644 --- a/src/ehrdata/io/source/vocab/loinc.py +++ b/src/ehrdata/io/source/vocab/loinc.py @@ -10,10 +10,13 @@ from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING import pandas as pd +if TYPE_CHECKING: + from pathlib import Path + def load_loinc_map(path: Path | str) -> pd.DataFrame: """Load a LOINC reference CSV into a lookup DataFrame. diff --git a/src/ehrdata/io/source/vocab/ndc.py b/src/ehrdata/io/source/vocab/ndc.py index 396820c6..6517f16c 100644 --- a/src/ehrdata/io/source/vocab/ndc.py +++ b/src/ehrdata/io/source/vocab/ndc.py @@ -11,10 +11,13 @@ from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING import pandas as pd +if TYPE_CHECKING: + from pathlib import Path + _NDC11_LEN = 11 diff --git a/src/ehrdata/io/source/vocab/prodcode.py b/src/ehrdata/io/source/vocab/prodcode.py index 3f45800f..824480c8 100644 --- a/src/ehrdata/io/source/vocab/prodcode.py +++ b/src/ehrdata/io/source/vocab/prodcode.py @@ -7,10 +7,13 @@ from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING import pandas as pd +if TYPE_CHECKING: + from pathlib import Path + def load_product_map(path: Path | str) -> pd.DataFrame: """Load CPRD ``product.txt`` or ``product.csv`` → (prodcode, drugsubstance). diff --git a/src/ehrdata/io/source/vocab/readcode.py b/src/ehrdata/io/source/vocab/readcode.py index eac691d1..20bd993e 100644 --- a/src/ehrdata/io/source/vocab/readcode.py +++ b/src/ehrdata/io/source/vocab/readcode.py @@ -6,10 +6,13 @@ from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING import pandas as pd +if TYPE_CHECKING: + from pathlib import Path + def load_medical_map(path: Path | str) -> pd.DataFrame: """Load CPRD ``medical.txt`` → (medcode, readcode) mapping. diff --git a/src/ehrdata/io/source/vocab/rxnorm.py b/src/ehrdata/io/source/vocab/rxnorm.py index f21b7a09..b33af02a 100644 --- a/src/ehrdata/io/source/vocab/rxnorm.py +++ b/src/ehrdata/io/source/vocab/rxnorm.py @@ -8,10 +8,13 @@ from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING import pandas as pd +if TYPE_CHECKING: + from pathlib import Path + def load_rxcui_ingredient_map(path: Path | str) -> pd.DataFrame: """Load the RxCUI-to-ingredient mapping file. diff --git a/tests/io/test_source_cprd.py b/tests/io/test_source_cprd.py index adc01c1c..200e0cd7 100644 --- a/tests/io/test_source_cprd.py +++ b/tests/io/test_source_cprd.py @@ -77,9 +77,10 @@ def test_dtype_is_string(self): def test_deduplicates_on_medcode(self): # feeding a path where the same medcode appears twice - import io data = "medcode\treadcode\tdesc\n100\tA10..00\tFoo\n100\tA10..00\tFoo duplicate\n" - import tempfile, os + import os + import tempfile + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: f.write(data) tmp = f.name @@ -155,8 +156,10 @@ def test_known_drugsubstance(self): assert row["drugsubstance"].iloc[0] == "metformin" def test_prefers_updated_column(self): - import io, tempfile, os - data = "prodcode\tdrugsubstance\tdrugsubstance.updated\n" "PC1\traw name\tupdated name\n" + import os + import tempfile + + data = "prodcode\tdrugsubstance\tdrugsubstance.updated\nPC1\traw name\tupdated name\n" with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: f.write(data) tmp = f.name @@ -167,7 +170,9 @@ def test_prefers_updated_column(self): os.unlink(tmp) def test_csv_separator(self): - import tempfile, os + import os + import tempfile + data = "prodcode,drugsubstance\nPC1,aspirin\n" with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: f.write(data) diff --git a/tests/io/test_source_extract.py b/tests/io/test_source_extract.py index 225552f5..ce268103 100644 --- a/tests/io/test_source_extract.py +++ b/tests/io/test_source_extract.py @@ -98,11 +98,13 @@ def test_correct_row_count(self): assert len(result) == 6 def test_deduplicates_identical_long_rows(self): - df = pd.DataFrame({ - "pid": [1, 1], - "dx1": ["A01", "A01"], - "dx2": ["B02", "B02"], - }) + df = pd.DataFrame( + { + "pid": [1, 1], + "dx1": ["A01", "A01"], + "dx2": ["B02", "B02"], + } + ) result = unnest_codes(df, id_cols=["pid"], code_cols=["dx1", "dx2"], value_name="dx") assert len(result) == 2 # (pid=1,dx=A01) and (pid=1,dx=B02) @@ -128,7 +130,7 @@ def test_index_reset(self): class TestReadZippedTsv: - @pytest.fixture() + @pytest.fixture def zip_bytes(self, tmp_path): content = "patient_id\teventdate\tdx\n1\t2020-01-15\tE11.9\n2\t2020-02-01\tI10\n" zdata = _make_zip({"clinical.txt": content}) @@ -157,7 +159,7 @@ def test_accepts_string_path(self, zip_bytes): class TestReadZippedTsvs: - @pytest.fixture() + @pytest.fixture def zip_bytes(self, tmp_path): tsv1 = "patient_id\tdx\n1\tE11.9\n2\tI10\n" tsv2 = "patient_id\tdx\n3\tE10.9\n4\tJ45\n" diff --git a/tests/io/test_source_lced.py b/tests/io/test_source_lced.py index 8d0c8fe4..df2d6bb6 100644 --- a/tests/io/test_source_lced.py +++ b/tests/io/test_source_lced.py @@ -33,17 +33,17 @@ # --------------------------------------------------------------------------- -@pytest.fixture() +@pytest.fixture def ndc_map(): return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") -@pytest.fixture() +@pytest.fixture def rxcui_map(): return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt") -@pytest.fixture() +@pytest.fixture def loinc_map(): return load_loinc_map(VOCAB_DIR / "loinc_map.csv") @@ -53,172 +53,195 @@ def loinc_map(): # --------------------------------------------------------------------------- -@pytest.fixture() +@pytest.fixture def facility_header(): - return pd.DataFrame({ - "patient_id": ["P001", "P002", "P001"], - "dxver": ["0", "9", "0"], - "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], - "dx1": ["E11.9", "I10", "E11.9"], - "dx2": ["I10", None, None], - "dx3": [None, None, None], - "proc1": ["99213", None, "99213"], - "proc2": [None, "93000", None], - "proc3": [None, None, None], - "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], - "cob": [0.0, 10.0, 0.0], - "coins": [20.0, 0.0, 20.0], - "copay": [30.0, 15.0, 30.0], - "dobyr": [1960, 1970, 1960], - "sex": ["M", "F", "M"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P002", "P001"], + "dxver": ["0", "9", "0"], + "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], + "dx1": ["E11.9", "I10", "E11.9"], + "dx2": ["I10", None, None], + "dx3": [None, None, None], + "proc1": ["99213", None, "99213"], + "proc2": [None, "93000", None], + "proc3": [None, None, None], + "cob": [0.0, 10.0, 0.0], + "coins": [20.0, 0.0, 20.0], + "copay": [30.0, 15.0, 30.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + } + ) + + +@pytest.fixture def inpatient_admissions(): - return pd.DataFrame({ - "patient_id": ["P001", "P003"], - "dxver": ["0", None], - "admdate": ["2020-04-01", "2020-05-15"], - "pdx": ["J18.9", "E10.9"], - "dx1": ["I10", None], - "dx2": [None, None], - "pproc": ["27447", None], - "proc1": [None, "99232"], - "dobyr": [1960, 1980], - "sex": ["M", "M"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P003"], + "dxver": ["0", None], + "admdate": ["2020-04-01", "2020-05-15"], + "pdx": ["J18.9", "E10.9"], + "dx1": ["I10", None], + "dx2": [None, None], + "pproc": ["27447", None], + "proc1": [None, "99232"], + "dobyr": [1960, 1980], + "sex": ["M", "M"], + } + ) + + +@pytest.fixture def inpatient_services(): - return pd.DataFrame({ - "patient_id": ["P001", "P002"], - "dxver": ["0", "0"], - "svcdate": ["2020-04-02", "2020-04-10"], - "pdx": ["J18.9", "K92.1"], - "dx1": ["I10", None], - "dx2": [None, None], - "proctyp": ["ICD", "CPT"], - "proc1": ["99232", "44950"], - "cob": [5.0, 0.0], - "coins": [10.0, 25.0], - "copay": [20.0, 40.0], - "dobyr": [1960, 1970], - "sex": ["M", "F"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P002"], + "dxver": ["0", "0"], + "svcdate": ["2020-04-02", "2020-04-10"], + "pdx": ["J18.9", "K92.1"], + "dx1": ["I10", None], + "dx2": [None, None], + "proctyp": ["ICD", "CPT"], + "proc1": ["99232", "44950"], + "cob": [5.0, 0.0], + "coins": [10.0, 25.0], + "copay": [20.0, 40.0], + "dobyr": [1960, 1970], + "sex": ["M", "F"], + } + ) + + +@pytest.fixture def lab_results(): - return pd.DataFrame({ - "patient_id": ["P001", "P002", "P003"], - "dxver": [None, "0", None], - "svcdate": ["2020-01-20", "2020-02-10", "2020-03-05"], - "dx1": ["V58.67", "E11.9", None], - "proctyp": ["CPT", "CPT", "CPT"], - "proc1": ["83036", "83036", None], - "result": ["7.2", "6.8", "5.4"], - "resltcat": [None, None, "normal"], - "resunit": ["mmol/L", "mmol/L", "mmol/L"], - "loinccd": ["14749-6", "14749-6", "14749-6"], - "dobyr": [1960, 1970, 1980], - "sex": ["M", "F", "M"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P002", "P003"], + "dxver": [None, "0", None], + "svcdate": ["2020-01-20", "2020-02-10", "2020-03-05"], + "dx1": ["V58.67", "E11.9", None], + "proctyp": ["CPT", "CPT", "CPT"], + "proc1": ["83036", "83036", None], + "result": ["7.2", "6.8", "5.4"], + "resltcat": [None, None, "normal"], + "resunit": ["mmol/L", "mmol/L", "mmol/L"], + "loinccd": ["14749-6", "14749-6", "14749-6"], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + } + ) + + +@pytest.fixture def outpatient_services(): - return pd.DataFrame({ - "patient_id": ["P002", "P003"], - "dxver": [None, "0"], - "svcdate": ["2020-06-01", "2020-07-01"], - "dx1": ["E10.9", "I10"], - "dx2": [None, None], - "proctyp": ["CPT", "CPT"], - "proc1": ["99213", "99214"], - "cob": [0.0, 0.0], - "coins": [15.0, 20.0], - "copay": [25.0, 35.0], - "dobyr": [1970, 1980], - "sex": ["F", "M"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P002", "P003"], + "dxver": [None, "0"], + "svcdate": ["2020-06-01", "2020-07-01"], + "dx1": ["E10.9", "I10"], + "dx2": [None, None], + "proctyp": ["CPT", "CPT"], + "proc1": ["99213", "99214"], + "cob": [0.0, 0.0], + "coins": [15.0, 20.0], + "copay": [25.0, 35.0], + "dobyr": [1970, 1980], + "sex": ["F", "M"], + } + ) + + +@pytest.fixture def v_drug(): - return pd.DataFrame({ - "patient_id": ["P001", "P002", "P001"], - "prescription_date": ["2020-01-10", "2020-02-05", "2020-01-10"], - "start_date": ["2020-01-15", "2020-02-10", "2020-01-15"], - "end_date": ["2020-04-15", "2020-05-10", "2020-04-15"], - "rx_cui": ["723", "4815", "723"], # metformin, insulin glargine - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P002", "P001"], + "prescription_date": ["2020-01-10", "2020-02-05", "2020-01-10"], + "start_date": ["2020-01-15", "2020-02-10", "2020-01-15"], + "end_date": ["2020-04-15", "2020-05-10", "2020-04-15"], + "rx_cui": ["723", "4815", "723"], # metformin, insulin glargine + } + ) + + +@pytest.fixture def v_outpatient_drug_claims(): - return pd.DataFrame({ - "patient_id": ["P001", "P003", "P002"], - "svcdate": ["2020-01-20", "2020-03-01", "2020-02-15"], - "daysupp": [30, 90, 30], - "refill": [0, 1, 0], - "ndcnum": ["00071015523", "00310751030", "00065063136"], - "cob": [0.0, 5.0, 0.0], - "coins": [10.0, 0.0, 20.0], - "copay": [5.0, 10.0, 15.0], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P003", "P002"], + "svcdate": ["2020-01-20", "2020-03-01", "2020-02-15"], + "daysupp": [30, 90, 30], + "refill": [0, 1, 0], + "ndcnum": ["00071015523", "00310751030", "00065063136"], + "cob": [0.0, 5.0, 0.0], + "coins": [10.0, 0.0, 20.0], + "copay": [5.0, 10.0, 15.0], + } + ) + + +@pytest.fixture def v_observation(): - return pd.DataFrame({ - "patient_id": ["P001", "P002"], - "observation_date": ["2020-01-15", "2020-02-01"], - "std_value": ["7.2", "6.8"], - "std_uom": ["mmol/L", "mmol/L"], - "loinc_test_id": ["14749-6", "4548-4"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "patient_id": ["P001", "P002"], + "observation_date": ["2020-01-15", "2020-02-01"], + "std_value": ["7.2", "6.8"], + "std_uom": ["mmol/L", "mmol/L"], + "loinc_test_id": ["14749-6", "4548-4"], + } + ) + + +@pytest.fixture def v_habit(): - return pd.DataFrame({ - "patient_id": ["P001", "P001", "P002", "P003"], - "mapped_question_answer": ["current smoker", None, "non-smoker", "former smoker"], - "encounter_join_id": [101, 102, 103, 104], - }) + return pd.DataFrame( + { + "patient_id": ["P001", "P001", "P002", "P003"], + "mapped_question_answer": ["current smoker", None, "non-smoker", "former smoker"], + "encounter_join_id": [101, 102, 103, 104], + } + ) -@pytest.fixture() +@pytest.fixture def v_encounter(): - return pd.DataFrame({ - "encounter_join_id": [101, 102, 103, 104], - "encounter_date": ["2020-01-15", "2020-02-01", "2020-02-10", "2020-03-01"], - }) + return pd.DataFrame( + { + "encounter_join_id": [101, 102, 103, 104], + "encounter_date": ["2020-01-15", "2020-02-01", "2020-02-10", "2020-03-01"], + } + ) -@pytest.fixture() +@pytest.fixture def v_annual_summary_enrollment(): - return pd.DataFrame({ - "patient_id": ["P001", "P002", "P003"], - "dobyr": [1960, 1970, 1980], - "sex": ["M", "F", "M"], - }) + return pd.DataFrame( + { + "patient_id": ["P001", "P002", "P003"], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + } + ) -@pytest.fixture() +@pytest.fixture def v_detail_enrollment(): - return pd.DataFrame({ - "patient_id": ["P001", "P001", "P002"], - "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], - "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], - "plantyp": [10, 10, 20], - "rx": ["Y", "Y", "N"], - "hlthplan": ["BlueCross", "BlueCross", "Aetna"], - "dobyr": [1960, 1960, 1970], - "sex": ["M", "M", "F"], - }) + return pd.DataFrame( + { + "patient_id": ["P001", "P001", "P002"], + "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], + "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], + "plantyp": [10, 10, 20], + "rx": ["Y", "Y", "N"], + "hlthplan": ["BlueCross", "BlueCross", "Aetna"], + "dobyr": [1960, 1960, 1970], + "sex": ["M", "M", "F"], + } + ) # =========================================================================== @@ -319,35 +342,63 @@ def test_classify_preserves_index(self): class TestLcedBuildDiagnosis: - def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_schema_valid( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert DIAGNOSIS.validate(result) == [] - def test_patient_id_already_named(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_patient_id_already_named( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert "patient_id" in result.columns assert "enrolid" not in result.columns - def test_no_null_dx_codes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_no_null_dx_codes( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert result["dx"].notna().all() - def test_five_sources_contribute(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_five_sources_contribute( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) # P003 only appears in inpatient_admissions and outpatient_services assert "P003" in result["patient_id"].values - def test_v_code_from_lab_results_inferred_icd9(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_v_code_from_lab_results_inferred_icd9( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) v_rows = result[(result["patient_id"] == "P001") & (result["dx"] == "V58.67")] assert (v_rows["dxver"] == "9").all() - def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_no_duplicate_rows( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert not result.duplicated().any() - def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_sorted_by_patient_then_date( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_diagnosis( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert list(result["patient_id"]) == sorted(result["patient_id"]) @@ -434,21 +485,37 @@ def test_sorted_by_patient_then_date(self, v_observation, lab_results): class TestLcedBuildProcedure: - def test_schema_valid(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_schema_valid( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_procedure( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert PROCEDURE.validate(result) == [] - def test_no_null_proc_codes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_no_null_proc_codes( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_procedure( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert result["proc"].notna().all() - def test_lab_results_source_contributes(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_lab_results_source_contributes( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_procedure( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) # lab_results has proc1=83036 for P001 and P002 assert "83036" in result["proc"].values - def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services): - result = build_procedure(facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services) + def test_no_duplicate_rows( + self, facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ): + result = build_procedure( + facility_header, inpatient_admissions, inpatient_services, lab_results, outpatient_services + ) assert not result.duplicated().any() @@ -513,11 +580,15 @@ def test_schema_valid(self, facility_header, inpatient_services, v_outpatient_dr result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) assert INSURANCE.validate(result) == [] - def test_patient_id_is_string(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + def test_patient_id_is_string( + self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services + ): result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) assert result["patient_id"].dtype == object - def test_svcdate_is_datetime(self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services): + def test_svcdate_is_datetime( + self, facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services + ): result = build_insurance(facility_header, inpatient_services, v_outpatient_drug_claims, outpatient_services) assert pd.api.types.is_datetime64_any_dtype(result["svcdate"]) diff --git a/tests/io/test_source_marketscan.py b/tests/io/test_source_marketscan.py index 429f36c6..384a02d8 100644 --- a/tests/io/test_source_marketscan.py +++ b/tests/io/test_source_marketscan.py @@ -29,191 +29,205 @@ # --------------------------------------------------------------------------- -@pytest.fixture() +@pytest.fixture def ndc_map(): return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") -@pytest.fixture() +@pytest.fixture def facility_header(): - return pd.DataFrame({ - "enrolid": [1001, 1002, 1001], - "dxver": ["0", "9", "0"], - "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], - "dx1": ["E11.9", "I10", "E11.9"], - "dx2": ["I10", None, None], - "dx3": [None, None, None], - "proc1": ["99213", None, "99213"], - "proc2": [None, "93000", None], - "proc3": [None, None, None], - "cob": [0.0, 10.0, 0.0], - "coins": [20.0, 0.0, 20.0], - "copay": [30.0, 15.0, 30.0], - "dobyr": [1960, 1970, 1960], - "sex": ["M", "F", "M"], - "efamid": [5001, 5002, 5001], - "year": [2020, 2020, 2020], - "region": ["NE", "SE", "NE"], - "msa": [10, 20, 10], - "wgtkey": [1, 2, 1], - "eeclass": ["A", "B", "A"], - "eestatu": ["1", "2", "1"], - "egeoloc": ["11", "22", "11"], - "emprel": ["1", "2", "1"], - "indstry": ["001", "002", "001"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1001, 1002, 1001], + "dxver": ["0", "9", "0"], + "svcdate": ["2020-01-15", "2020-02-01", "2020-03-10"], + "dx1": ["E11.9", "I10", "E11.9"], + "dx2": ["I10", None, None], + "dx3": [None, None, None], + "proc1": ["99213", None, "99213"], + "proc2": [None, "93000", None], + "proc3": [None, None, None], + "cob": [0.0, 10.0, 0.0], + "coins": [20.0, 0.0, 20.0], + "copay": [30.0, 15.0, 30.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5001], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "NE"], + "msa": [10, 20, 10], + "wgtkey": [1, 2, 1], + "eeclass": ["A", "B", "A"], + "eestatu": ["1", "2", "1"], + "egeoloc": ["11", "22", "11"], + "emprel": ["1", "2", "1"], + "indstry": ["001", "002", "001"], + } + ) + + +@pytest.fixture def inpatient_admissions(): - return pd.DataFrame({ - "enrolid": [1001, 1003], - "dxver": ["0", None], - "admdate": ["2020-04-01", "2020-05-15"], - "pdx": ["J18.9", "E10.9"], - "dx1": ["I10", None], - "dx2": [None, None], - "pproc": ["27447", None], - "proc1": [None, "99232"], - "dobyr": [1960, 1980], - "sex": ["M", "M"], - "efamid": [5001, 5003], - "year": [2020, 2020], - "region": ["NE", "MW"], - "msa": [10, 30], - "wgtkey": [1, 3], - "eeclass": ["A", "C"], - "eestatu": ["1", "3"], - "egeoloc": ["11", "33"], - "emprel": ["1", "3"], - "indstry": ["001", "003"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1001, 1003], + "dxver": ["0", None], + "admdate": ["2020-04-01", "2020-05-15"], + "pdx": ["J18.9", "E10.9"], + "dx1": ["I10", None], + "dx2": [None, None], + "pproc": ["27447", None], + "proc1": [None, "99232"], + "dobyr": [1960, 1980], + "sex": ["M", "M"], + "efamid": [5001, 5003], + "year": [2020, 2020], + "region": ["NE", "MW"], + "msa": [10, 30], + "wgtkey": [1, 3], + "eeclass": ["A", "C"], + "eestatu": ["1", "3"], + "egeoloc": ["11", "33"], + "emprel": ["1", "3"], + "indstry": ["001", "003"], + } + ) + + +@pytest.fixture def inpatient_services(): - return pd.DataFrame({ - "enrolid": [1001, 1002], - "dxver": ["0", "0"], - "svcdate": ["2020-04-02", "2020-04-10"], - "pdx": ["J18.9", "K92.1"], - "dx1": ["I10", None], - "dx2": [None, None], - "proctyp": ["ICD", "CPT"], - "proc1": ["99232", "44950"], - "cob": [5.0, 0.0], - "coins": [10.0, 25.0], - "copay": [20.0, 40.0], - "dobyr": [1960, 1970], - "sex": ["M", "F"], - "efamid": [5001, 5002], - "year": [2020, 2020], - "region": ["NE", "SE"], - "msa": [10, 20], - "wgtkey": [1, 2], - "eeclass": ["A", "B"], - "eestatu": ["1", "2"], - "egeoloc": ["11", "22"], - "emprel": ["1", "2"], - "indstry": ["001", "002"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1001, 1002], + "dxver": ["0", "0"], + "svcdate": ["2020-04-02", "2020-04-10"], + "pdx": ["J18.9", "K92.1"], + "dx1": ["I10", None], + "dx2": [None, None], + "proctyp": ["ICD", "CPT"], + "proc1": ["99232", "44950"], + "cob": [5.0, 0.0], + "coins": [10.0, 25.0], + "copay": [20.0, 40.0], + "dobyr": [1960, 1970], + "sex": ["M", "F"], + "efamid": [5001, 5002], + "year": [2020, 2020], + "region": ["NE", "SE"], + "msa": [10, 20], + "wgtkey": [1, 2], + "eeclass": ["A", "B"], + "eestatu": ["1", "2"], + "egeoloc": ["11", "22"], + "emprel": ["1", "2"], + "indstry": ["001", "002"], + } + ) + + +@pytest.fixture def outpatient_services(): - return pd.DataFrame({ - "enrolid": [1002, 1003], - "dxver": [None, "0"], - "svcdate": ["2020-06-01", "2020-07-01"], - "dx1": ["E10.9", "I10"], - "dx2": [None, None], - "proctyp": ["CPT", "CPT"], - "proc1": ["99213", "99214"], - "cob": [0.0, 0.0], - "coins": [15.0, 20.0], - "copay": [25.0, 35.0], - "dobyr": [1970, 1980], - "sex": ["F", "M"], - "efamid": [5002, 5003], - "year": [2020, 2020], - "region": ["SE", "MW"], - "msa": [20, 30], - "wgtkey": [2, 3], - "eeclass": ["B", "C"], - "eestatu": ["2", "3"], - "egeoloc": ["22", "33"], - "emprel": ["2", "3"], - "indstry": ["002", "003"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1002, 1003], + "dxver": [None, "0"], + "svcdate": ["2020-06-01", "2020-07-01"], + "dx1": ["E10.9", "I10"], + "dx2": [None, None], + "proctyp": ["CPT", "CPT"], + "proc1": ["99213", "99214"], + "cob": [0.0, 0.0], + "coins": [15.0, 20.0], + "copay": [25.0, 35.0], + "dobyr": [1970, 1980], + "sex": ["F", "M"], + "efamid": [5002, 5003], + "year": [2020, 2020], + "region": ["SE", "MW"], + "msa": [20, 30], + "wgtkey": [2, 3], + "eeclass": ["B", "C"], + "eestatu": ["2", "3"], + "egeoloc": ["22", "33"], + "emprel": ["2", "3"], + "indstry": ["002", "003"], + } + ) + + +@pytest.fixture def outpatient_prescription_drugs(): - return pd.DataFrame({ - "enrolid": [1001, 1002, 1001], - "svcdate": ["2020-01-20", "2020-02-05", "2020-01-20"], - "daysupp": [30, 90, 30], - "refill": [0, 1, 0], - "ndcnum": ["00071015523", "00310751030", "00071015523"], # metformin, insulin glargine - "cob": [0.0, 5.0, 0.0], - "coins": [10.0, 0.0, 10.0], - "copay": [5.0, 10.0, 5.0], - "dobyr": [1960, 1970, 1960], - "sex": ["M", "F", "M"], - "efamid": [5001, 5002, 5001], - "year": [2020, 2020, 2020], - "region": ["NE", "SE", "NE"], - "msa": [10, 20, 10], - "wgtkey": [1, 2, 1], - "eeclass": ["A", "B", "A"], - "eestatu": ["1", "2", "1"], - "egeoloc": ["11", "22", "11"], - "emprel": ["1", "2", "1"], - "indstry": ["001", "002", "001"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1001, 1002, 1001], + "svcdate": ["2020-01-20", "2020-02-05", "2020-01-20"], + "daysupp": [30, 90, 30], + "refill": [0, 1, 0], + "ndcnum": ["00071015523", "00310751030", "00071015523"], # metformin, insulin glargine + "cob": [0.0, 5.0, 0.0], + "coins": [10.0, 0.0, 10.0], + "copay": [5.0, 10.0, 5.0], + "dobyr": [1960, 1970, 1960], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5001], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "NE"], + "msa": [10, 20, 10], + "wgtkey": [1, 2, 1], + "eeclass": ["A", "B", "A"], + "eestatu": ["1", "2", "1"], + "egeoloc": ["11", "22", "11"], + "emprel": ["1", "2", "1"], + "indstry": ["001", "002", "001"], + } + ) + + +@pytest.fixture def enrollment_annual_summary(): - return pd.DataFrame({ - "enrolid": [1001, 1002, 1003], - "dobyr": [1960, 1970, 1980], - "sex": ["M", "F", "M"], - "efamid": [5001, 5002, 5003], - "year": [2020, 2020, 2020], - "region": ["NE", "SE", "MW"], - "msa": [10, 20, 30], - "wgtkey": [1, 2, 3], - "eeclass": ["A", "B", "C"], - "eestatu": ["1", "2", "3"], - "egeoloc": ["11", "22", "33"], - "emprel": ["1", "2", "3"], - "indstry": ["001","002","003"], - }) - - -@pytest.fixture() + return pd.DataFrame( + { + "enrolid": [1001, 1002, 1003], + "dobyr": [1960, 1970, 1980], + "sex": ["M", "F", "M"], + "efamid": [5001, 5002, 5003], + "year": [2020, 2020, 2020], + "region": ["NE", "SE", "MW"], + "msa": [10, 20, 30], + "wgtkey": [1, 2, 3], + "eeclass": ["A", "B", "C"], + "eestatu": ["1", "2", "3"], + "egeoloc": ["11", "22", "33"], + "emprel": ["1", "2", "3"], + "indstry": ["001", "002", "003"], + } + ) + + +@pytest.fixture def enrollment_detail(): - return pd.DataFrame({ - "enrolid": [1001, 1001, 1002], - "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], - "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], - "plantyp": [10, 10, 20], - "rx": ["Y", "Y", "N"], - "hlthplan": ["BlueCross", "BlueCross", "Aetna"], - "dobyr": [1960, 1960, 1970], - "sex": ["M", "M", "F"], - "efamid": [5001, 5001, 5002], - "year": [2020, 2021, 2020], - "region": ["NE", "NE", "SE"], - "msa": [10, 10, 20], - "wgtkey": [1, 1, 2], - "eeclass": ["A", "A", "B"], - "eestatu": ["1", "1", "2"], - "egeoloc": ["11", "11", "22"], - "emprel": ["1", "1", "2"], - "indstry": ["001", "001", "002"], - }) + return pd.DataFrame( + { + "enrolid": [1001, 1001, 1002], + "dtstart": ["2020-01-01", "2021-01-01", "2020-01-01"], + "dtend": ["2020-12-31", "2021-12-31", "2020-12-31"], + "plantyp": [10, 10, 20], + "rx": ["Y", "Y", "N"], + "hlthplan": ["BlueCross", "BlueCross", "Aetna"], + "dobyr": [1960, 1960, 1970], + "sex": ["M", "M", "F"], + "efamid": [5001, 5001, 5002], + "year": [2020, 2021, 2020], + "region": ["NE", "NE", "SE"], + "msa": [10, 10, 20], + "wgtkey": [1, 1, 2], + "eeclass": ["A", "A", "B"], + "eestatu": ["1", "1", "2"], + "egeoloc": ["11", "11", "22"], + "emprel": ["1", "1", "2"], + "indstry": ["001", "001", "002"], + } + ) # --------------------------------------------------------------------------- @@ -230,7 +244,9 @@ def test_patient_id_is_string(self, facility_header, inpatient_admissions, inpat result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert result["patient_id"].dtype == object - def test_eventdate_is_datetime(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_eventdate_is_datetime( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) @@ -238,7 +254,9 @@ def test_no_null_dx_codes(self, facility_header, inpatient_admissions, inpatient result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert result["dx"].notna().all() - def test_enrolid_renamed_to_patient_id(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_enrolid_renamed_to_patient_id( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert "patient_id" in result.columns assert "enrolid" not in result.columns @@ -247,12 +265,16 @@ def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatien result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert not result.duplicated().any() - def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_sorted_by_patient_then_date( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) patients = list(result["patient_id"]) assert patients == sorted(patients) - def test_icd_version_inferred_for_missing(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_icd_version_inferred_for_missing( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): # outpatient_services has dxver=None for enrolid=1002 with dx=E10.9 result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) e10_rows = result[(result["patient_id"] == "1002") & (result["dx"] == "E10.9")] @@ -266,7 +288,9 @@ def test_wide_dx_unnested(self, facility_header, inpatient_admissions, inpatient assert "E11.9" in patient_1001_dx assert "I10" in patient_1001_dx - def test_all_four_sources_contribute(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_all_four_sources_contribute( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_diagnosis(facility_header, inpatient_admissions, inpatient_services, outpatient_services) # Patient 1003 only appears in inpatient_admissions and outpatient_services assert "1003" in result["patient_id"].values @@ -348,18 +372,24 @@ def test_enrolid_renamed(self, facility_header, inpatient_admissions, inpatient_ assert "patient_id" in result.columns assert "enrolid" not in result.columns - def test_eventdate_is_datetime(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_eventdate_is_datetime( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert pd.api.types.is_datetime64_any_dtype(result["eventdate"]) - def test_proctype_null_for_facility_header(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_proctype_null_for_facility_header( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) # facility_header contributions should have null proctype fh_date = pd.Timestamp("2020-01-15") fh_rows = result[(result["patient_id"] == "1001") & (result["eventdate"] == fh_date)] assert fh_rows["proctype"].isna().all() - def test_proctype_set_for_outpatient_services(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_proctype_set_for_outpatient_services( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) op_rows = result[result["patient_id"] == "1002"] cpt_rows = op_rows[op_rows["proctype"] == "CPT"] @@ -369,7 +399,9 @@ def test_no_duplicate_rows(self, facility_header, inpatient_admissions, inpatien result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) assert not result.duplicated().any() - def test_sorted_by_patient_then_date(self, facility_header, inpatient_admissions, inpatient_services, outpatient_services): + def test_sorted_by_patient_then_date( + self, facility_header, inpatient_admissions, inpatient_services, outpatient_services + ): result = build_procedure(facility_header, inpatient_admissions, inpatient_services, outpatient_services) patients = list(result["patient_id"]) assert patients == sorted(patients) @@ -412,7 +444,8 @@ def test_single_source(self, enrollment_annual_summary): def test_all_patients_present(self, enrollment_annual_summary, facility_header): result = build_patinfo(enrollment_annual_summary, facility_header) pids = set(result["patient_id"].tolist()) - assert "1001" in pids and "1002" in pids + assert "1001" in pids + assert "1002" in pids # --------------------------------------------------------------------------- @@ -421,26 +454,46 @@ def test_all_patients_present(self, enrollment_annual_summary, facility_header): class TestBuildInsurance: - def test_schema_valid(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): - result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + def test_schema_valid( + self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ): + result = build_insurance( + facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ) assert INSURANCE.validate(result) == [] - def test_enrolid_renamed(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): - result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + def test_enrolid_renamed( + self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ): + result = build_insurance( + facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ) assert "patient_id" in result.columns assert "enrolid" not in result.columns - def test_svcdate_is_datetime(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): - result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + def test_svcdate_is_datetime( + self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ): + result = build_insurance( + facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ) assert pd.api.types.is_datetime64_any_dtype(result["svcdate"]) - def test_cob_coins_copay_numeric(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): - result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + def test_cob_coins_copay_numeric( + self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ): + result = build_insurance( + facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ) for col in ("cob", "coins", "copay"): assert pd.api.types.is_float_dtype(result[col]) - def test_no_duplicates(self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services): - result = build_insurance(facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services) + def test_no_duplicates( + self, facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ): + result = build_insurance( + facility_header, inpatient_services, outpatient_prescription_drugs, outpatient_services + ) assert not result.duplicated().any() diff --git a/tests/io/test_source_normalize.py b/tests/io/test_source_normalize.py index 47ce10ee..98d8c67a 100644 --- a/tests/io/test_source_normalize.py +++ b/tests/io/test_source_normalize.py @@ -125,10 +125,10 @@ def test_does_not_mutate_input(self): def test_mixed_rows(self): df = self._make_df([None, "0", None, None], ["E930.0", "E11.9", "V58.67", "I10"]) result = infer_icd_version(df) - assert result.loc[0, "dxver"] == "9" # E prefix + null → inferred - assert result.loc[1, "dxver"] == "0" # explicit, not overwritten - assert result.loc[2, "dxver"] == "9" # V prefix + null → inferred - assert pd.isna(result.loc[3, "dxver"]) # I prefix + null → stays null + assert result.loc[0, "dxver"] == "9" # E prefix + null → inferred + assert result.loc[1, "dxver"] == "0" # explicit, not overwritten + assert result.loc[2, "dxver"] == "9" # V prefix + null → inferred + assert pd.isna(result.loc[3, "dxver"]) # I prefix + null → stays null def test_custom_column_names(self): df = pd.DataFrame({"pid": [1], "ver": [None], "code": ["E930.0"]}) @@ -170,28 +170,38 @@ def test_no_duplicates_unchanged(self): class TestSortEvents: def test_sorted_by_patient_then_date(self): - df = pd.DataFrame({ - "patient_id": ["2", "1", "1"], - "eventdate": pd.to_datetime(["2020-02-01", "2020-03-01", "2020-01-01"]), - }) + df = pd.DataFrame( + { + "patient_id": ["2", "1", "1"], + "eventdate": pd.to_datetime(["2020-02-01", "2020-03-01", "2020-01-01"]), + } + ) result = sort_events(df) assert list(result["patient_id"]) == ["1", "1", "2"] - assert list(result["eventdate"]) == [pd.Timestamp("2020-01-01"), pd.Timestamp("2020-03-01"), pd.Timestamp("2020-02-01")] + assert list(result["eventdate"]) == [ + pd.Timestamp("2020-01-01"), + pd.Timestamp("2020-03-01"), + pd.Timestamp("2020-02-01"), + ] def test_nat_placed_last(self): - df = pd.DataFrame({ - "patient_id": ["1", "1"], - "eventdate": pd.to_datetime([None, "2020-01-01"]), - }) + df = pd.DataFrame( + { + "patient_id": ["1", "1"], + "eventdate": pd.to_datetime([None, "2020-01-01"]), + } + ) result = sort_events(df) assert result["eventdate"].iloc[0] == pd.Timestamp("2020-01-01") assert pd.isna(result["eventdate"].iloc[1]) def test_custom_column_names(self): - df = pd.DataFrame({ - "pid": ["b", "a"], - "dt": pd.to_datetime(["2020-02-01", "2020-01-01"]), - }) + df = pd.DataFrame( + { + "pid": ["b", "a"], + "dt": pd.to_datetime(["2020-02-01", "2020-01-01"]), + } + ) result = sort_events(df, patient_col="pid", date_col="dt") assert result["pid"].iloc[0] == "a" @@ -202,7 +212,7 @@ def test_custom_column_names(self): class TestNormalizeDiagnosis: - @pytest.fixture() + @pytest.fixture def raw_df(self): return pd.read_csv(FIXTURE_DIR / "diagnosis.csv") @@ -254,7 +264,7 @@ def test_does_not_mutate_input(self, raw_df): class TestNormalizeTherapy: - @pytest.fixture() + @pytest.fixture def raw_df(self): return pd.read_csv(FIXTURE_DIR / "therapy.csv") @@ -289,7 +299,7 @@ def test_patient_id_is_string(self, raw_df): class TestNormalizeLabtest: - @pytest.fixture() + @pytest.fixture def raw_df(self): return pd.read_csv(FIXTURE_DIR / "labtest.csv") @@ -315,7 +325,7 @@ def test_sorted_by_patient_then_date(self, raw_df): class TestNormalizeProcedure: - @pytest.fixture() + @pytest.fixture def raw_df(self): return pd.read_csv(FIXTURE_DIR / "procedure.csv") diff --git a/tests/io/test_source_schema.py b/tests/io/test_source_schema.py index b74563e0..68beb775 100644 --- a/tests/io/test_source_schema.py +++ b/tests/io/test_source_schema.py @@ -12,7 +12,6 @@ PROVIDER, THERAPY, ColumnSpec, - TableSchema, ) @@ -53,7 +52,9 @@ def test_empty_datetime_dtype(self): assert pd.api.types.is_datetime64_any_dtype(df["eventdate"]) def test_validate_valid_dataframe(self): - df = pd.DataFrame({"patient_id": ["1"], "dxver": ["0"], "eventdate": pd.to_datetime(["2020-01-01"]), "dx": ["E11.9"]}) + df = pd.DataFrame( + {"patient_id": ["1"], "dxver": ["0"], "eventdate": pd.to_datetime(["2020-01-01"]), "dx": ["E11.9"]} + ) assert DIAGNOSIS.validate(df) == [] def test_validate_missing_column(self): @@ -68,11 +69,15 @@ def test_validate_multiple_missing_columns(self): assert len(errors) == 3 def test_validate_strict_rejects_extra_columns(self): - df = pd.DataFrame({ - "patient_id": ["1"], "dxver": ["0"], - "eventdate": pd.to_datetime(["2020-01-01"]), - "dx": ["E11.9"], "extra_col": [99], - }) + df = pd.DataFrame( + { + "patient_id": ["1"], + "dxver": ["0"], + "eventdate": pd.to_datetime(["2020-01-01"]), + "dx": ["E11.9"], + "extra_col": [99], + } + ) assert DIAGNOSIS.validate(df, strict=False) == [] errors = DIAGNOSIS.validate(df, strict=True) assert len(errors) == 1 @@ -85,7 +90,16 @@ def test_validate_strict_no_false_positives(self): class TestAllSchemas: def test_contains_all_eight_tables(self): - assert set(ALL_SCHEMAS.keys()) == {"diagnosis", "therapy", "labtest", "procedure", "patinfo", "insurance", "provider", "habit"} + assert set(ALL_SCHEMAS.keys()) == { + "diagnosis", + "therapy", + "labtest", + "procedure", + "patinfo", + "insurance", + "provider", + "habit", + } def test_all_schemas_start_with_patient_id(self): for name, schema in ALL_SCHEMAS.items(): @@ -95,15 +109,31 @@ def test_all_schemas_have_non_empty_columns(self): for name, schema in ALL_SCHEMAS.items(): assert len(schema.columns) > 0, f"{name} has no columns" - @pytest.mark.parametrize("schema,expected_cols", [ - (THERAPY, ["patient_id", "prescription_date", "start_date", "fill_date", "end_date", "refill", "rxcui", "ndc11", "ingredient"]), - (LABTEST, ["patient_id", "eventdate", "value", "valuecat", "unit", "loinc"]), - (PROCEDURE, ["patient_id", "proctype", "eventdate", "proc"]), - (PATINFO, ["patient_id", "dobyr", "sex"]), - (INSURANCE, ["patient_id", "svcdate", "cob", "coins", "copay"]), - (PROVIDER, ["patient_id", "dtstart", "dtend", "plantyp", "rx", "hlthplan"]), - (HABIT, ["patient_id", "encounter_date", "mapped_question_answer"]), - ]) + @pytest.mark.parametrize( + ("schema", "expected_cols"), + [ + ( + THERAPY, + [ + "patient_id", + "prescription_date", + "start_date", + "fill_date", + "end_date", + "refill", + "rxcui", + "ndc11", + "ingredient", + ], + ), + (LABTEST, ["patient_id", "eventdate", "value", "valuecat", "unit", "loinc"]), + (PROCEDURE, ["patient_id", "proctype", "eventdate", "proc"]), + (PATINFO, ["patient_id", "dobyr", "sex"]), + (INSURANCE, ["patient_id", "svcdate", "cob", "coins", "copay"]), + (PROVIDER, ["patient_id", "dtstart", "dtend", "plantyp", "rx", "hlthplan"]), + (HABIT, ["patient_id", "encounter_date", "mapped_question_answer"]), + ], + ) def test_schema_columns(self, schema, expected_cols): assert schema.column_names == expected_cols diff --git a/tests/io/test_source_to_ehrdata.py b/tests/io/test_source_to_ehrdata.py index a4b0729b..b853deb9 100644 --- a/tests/io/test_source_to_ehrdata.py +++ b/tests/io/test_source_to_ehrdata.py @@ -8,69 +8,77 @@ import numpy as np import pandas as pd -import pytest from ehrdata.io.source.to_ehrdata import to_ehrdata - # --------------------------------------------------------------------------- # Minimal fixture helpers # --------------------------------------------------------------------------- def _patinfo(*patient_ids: str) -> pd.DataFrame: - return pd.DataFrame({ - "patient_id": list(patient_ids), - "dobyr": [1960 + i for i in range(len(patient_ids))], - "sex": ["F" if i % 2 == 0 else "M" for i in range(len(patient_ids))], - }) + return pd.DataFrame( + { + "patient_id": list(patient_ids), + "dobyr": [1960 + i for i in range(len(patient_ids))], + "sex": ["F" if i % 2 == 0 else "M" for i in range(len(patient_ids))], + } + ) def _diagnosis(rows: list[tuple[str, str]]) -> pd.DataFrame: - pids, dxs = zip(*rows) if rows else ([], []) - return pd.DataFrame({ - "patient_id": list(pids), - "dxver": [None] * len(pids), - "eventdate": pd.NaT, - "dx": list(dxs), - }) + pids, dxs = zip(*rows, strict=False) if rows else ([], []) + return pd.DataFrame( + { + "patient_id": list(pids), + "dxver": [None] * len(pids), + "eventdate": pd.NaT, + "dx": list(dxs), + } + ) def _therapy(rows: list[tuple[str, str | None]]) -> pd.DataFrame: - pids, ings = zip(*rows) if rows else ([], []) - return pd.DataFrame({ - "patient_id": list(pids), - "prescription_date": pd.NaT, - "start_date": pd.NaT, - "fill_date": pd.NaT, - "end_date": pd.NaT, - "refill": pd.array([pd.NA] * len(pids), dtype="Int64"), - "rxcui": None, - "ndc11": None, - "ingredient": list(ings), - }) + pids, ings = zip(*rows, strict=False) if rows else ([], []) + return pd.DataFrame( + { + "patient_id": list(pids), + "prescription_date": pd.NaT, + "start_date": pd.NaT, + "fill_date": pd.NaT, + "end_date": pd.NaT, + "refill": pd.array([pd.NA] * len(pids), dtype="Int64"), + "rxcui": None, + "ndc11": None, + "ingredient": list(ings), + } + ) def _labtest(rows: list[tuple[str, str | None]]) -> pd.DataFrame: - pids, loincs = zip(*rows) if rows else ([], []) - return pd.DataFrame({ - "patient_id": list(pids), - "eventdate": pd.NaT, - "value": None, - "valuecat": None, - "unit": None, - "loinc": list(loincs), - }) + pids, loincs = zip(*rows, strict=False) if rows else ([], []) + return pd.DataFrame( + { + "patient_id": list(pids), + "eventdate": pd.NaT, + "value": None, + "valuecat": None, + "unit": None, + "loinc": list(loincs), + } + ) def _procedure(rows: list[tuple[str, str]]) -> pd.DataFrame: - pids, procs = zip(*rows) if rows else ([], []) - return pd.DataFrame({ - "patient_id": list(pids), - "proctype": None, - "eventdate": pd.NaT, - "proc": list(procs), - }) + pids, procs = zip(*rows, strict=False) if rows else ([], []) + return pd.DataFrame( + { + "patient_id": list(pids), + "proctype": None, + "eventdate": pd.NaT, + "proc": list(procs), + } + ) # --------------------------------------------------------------------------- diff --git a/tests/io/test_source_vocab.py b/tests/io/test_source_vocab.py index c5c3f359..85e44b92 100644 --- a/tests/io/test_source_vocab.py +++ b/tests/io/test_source_vocab.py @@ -15,7 +15,7 @@ class TestLoadNdcIngredientMap: - @pytest.fixture() + @pytest.fixture def ndc_map(self): return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") @@ -49,7 +49,7 @@ def test_accepts_string_path(self): class TestJoinIngredientByNdc: - @pytest.fixture() + @pytest.fixture def ndc_map(self): return load_ndc_ingredient_map(VOCAB_DIR / "ndc_ingredient_map.txt") @@ -97,7 +97,7 @@ def test_duplicate_ndc_in_map_no_explosion(self, ndc_map): class TestLoadRxcuiIngredientMap: - @pytest.fixture() + @pytest.fixture def rxcui_map(self): return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt") @@ -122,7 +122,7 @@ def test_rxcui_is_string(self, rxcui_map): class TestJoinIngredientByRxcui: - @pytest.fixture() + @pytest.fixture def rxcui_map(self): return load_rxcui_ingredient_map(VOCAB_DIR / "rxcui_ingredient_map.txt")