diff --git a/MANIFEST.in b/MANIFEST.in index 452a986..43811cc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,4 @@ recursive-include tests/data/additional_svs *.bam *.bai *.md recursive-include tests/data/comprehensive *.json.gz *.md recursive-include tests/data/evidence-overlap *.bam *.md recursive-include tests/data/retrieval *.bam *.bai *.md +recursive-include tests/data/otud4-review *.fasta *.vcf *.md diff --git a/docs/changelog.md b/docs/changelog.md index a5c4c16..b062a19 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,24 @@ your version with `osteosarc --version`. Pin both the package version and your snapshot (by name or download date) for reproducible analyses. Full release notes are on [GitHub](https://github.com/iskandr/osteosarc/releases). +## Unreleased + +- Link FASTQs from published filename prefixes and the Tempus tumor/normal row + aliases, count only raw-read files, and generate working download commands. + FASTQ library metadata supplies assay/provider filters even for shared folders + such as CeGaT RNA delivered inside a WES directory. + Refresh the sample guide against all 31 samples in the October 7 metadata. + Data-page provider notes and mixed DNA/RNA analysis directories no longer + introduce false assay or provider conflicts. +- Supplementary-depth probes stream record counts and cache small receipts instead + of entire BAM extracts. Existing verified probes and overflow results are reused; + the depth cap and selected fixture records are unchanged. Reused BAM counts are + checked against their original records, including compact acquisition receipts. +- Recognize the reviewed 2026-10-05 upstream allele, ID, specimen and timeline + corrections while preserving historical snapshots and future drift checks. + A public CeGaT normal-DNA call supplies OTUD4's newly published literal allele; + the original Natera report and somatic status remain unverified. + ## 0.15.4 - Bounded extraction reads BAM pipes without constructing a pysam AlignmentFile diff --git a/docs/corrections.md b/docs/corrections.md index 3c2809b..7661d94 100644 --- a/docs/corrections.md +++ b/docs/corrections.md @@ -90,8 +90,12 @@ that should still be there, since a record disappearing doesn't prove a fix. ## The corrections -Checked against the website on 2026-09-24: 30 fixes apply and 5 are already fixed on -the site; a snapshot from 2026-09-18 uses all 35. Each fix's evidence is in +Reviewed again against the website on 2026-10-05. Fifteen additional layouts accept +verified upstream changes, including renamed TRMO, literal MAP2/DCHS2/GAPVD1 +alleles, the removed split MAP2 entries, the expanded FAM157A protein caveat, +corrected specimen descriptions, T0 Tempus tumor labels and ReyaGel's explicit end +year. Historical snapshots retain their original corrections and unexpected +future values still warn. Each fix's evidence is in `data.corrections` and the [source](https://github.com/iskandr/osteosarc/blob/main/osteosarc/curation.py). None only rewrites a value in an equivalent form: each fixes a wrong value, fills in a @@ -114,8 +118,8 @@ missing one, or flags something to be careful with. | muc3a-grch38-placement | note | The Tempus call is on GRCh37, in a repeat that differs between builds, so its GRCh38 position stays unresolved. | | ush2a-transposed-duplicate | note | USH2A-chr1-215560752 is a typo of USH2A-chr1-215650752. The site has merged them; the kept entry's location label and sequence context are fixed. | | fam157a-withdrawn-protein | flag | NCBI withdrew the protein model behind FAM157A's insertion; the site now says so too. | -| natera-alleles-unavailable | flag | COL3A1 and OTUD4 come from a Natera report that isn't public; a Tempus call now gives COL3A1's allele. | -| otud4-source-unavailable | note | No public call gives OTUD4's allele, so the entry has none. | +| natera-alleles-unavailable | flag | The original Natera report is not public. Independent Tempus and CeGaT calls supply matching COL3A1 and OTUD4 alleles; they do not verify that report or establish somatic status. | +| otud4-source-unavailable | note | Historical entries remain unresolved. The reviewed 2026-10-05 entry matches a public CeGaT normal-DNA deletion call, with transcript and assembly checks; this does not verify the original Natera call or somatic status. | | transcript-DCHS2 | edit | NM_1142552 is missing two zeros: NM_001142552, which the site now uses. | | transcript-COL4A2 | edit | NM_001846. has a stray dot. | | transcript-GTF3C5 | edit | NM_00112283 is missing a digit: NM_001122823. | @@ -153,10 +157,10 @@ cleared rather than read as zero. | specimen-T3-site | edit | T3 was an MSKCC resection, not a UCSF biopsy. | | pbmc-capture-dates | flag | Four blood samples are dated by cell capture; the blood was drawn two to four days earlier. | | events-duplicate-rows | flag | SQ3370 and Trabectedin each appear twice. | -| tempus-timepoint | flag | The timeline dates the Tempus tests to T0 (2022), which the data supports, but the files are labeled T1 (2024-06). | -| tempus-file-labels | flag | The Tempus files are labeled T1, but carry all three variants seen only at T0 and none of the 35 seen only at T1. The labels are left as published. | +| tempus-timepoint | flag | The timeline dates the Tempus tests to T0 (2022), consistent with the data and the reviewed 2026-10-05 tumor-file labels. Historical file labels say T1 (2024-06). | +| tempus-file-labels | flag | Historical Tempus files are labeled T1, but carry T0 evidence. The reviewed 2026-10-05 metadata labels tumor products T0 and leaves the later normal draw without a tumor timepoint. Published labels are retained; overlapping RNA products remain dependent evidence. | | apheresis-date | flag | The apheresis is 2024-05-14 on the timeline and 2024-05-15 in the ELISPOT records. | -| reyagel-end-date | flag | ReyaGel's end date is 7/14, with no year, so it shows as one day; the only broken date of the sheet's 371. | +| reyagel-end-date | flag | Historical ReyaGel's end date is 7/14 without a year. The reviewed 2026-10-05 sheet and event explicitly give 2026-07-14. | ## Other things to know diff --git a/docs/reads.md b/docs/reads.md index 2646615..c3ce8e2 100644 --- a/docs/reads.md +++ b/docs/reads.md @@ -1,5 +1,12 @@ # Reads +For a depth check without retaining a BAM, use +`data.count_reads(file, regions, max_records=1_000_000)`. The result has +`records` and an acquisition `receipt`. It counts records in the indexed region +union, including original duplicates, and does not fetch mates. Exceeding the cap +raises `RecordLimitError`; complete counts and overflow results are cached for +offline reuse. The standalone equivalent is `osteosarc.count_reads(...)`. + Copy just the reads you need out of the remote BAMs into small local BAMs: test data for a unit test in seconds, without downloading any BAM whole. You need SAMtools 1.21 or newer on your PATH; reopening reads you already fetched diff --git a/docs/samples.md b/docs/samples.md index ac5cf51..8db8e15 100644 --- a/docs/samples.md +++ b/docs/samples.md @@ -21,19 +21,21 @@ overlap is known, not proof of independence. The published FASTQ catalogue already identifies KCVBE1UI1P as a duplicate delivery of ALMY2X4KMV RNA. Corresponding mate objects also have matching lengths and multipart ETags (server metadata, not whole-file SHA-256 hashes). The overlap registry preserves -that source receipt and the object metadata. The existing `tempus-file-labels` -curation disputes the published T1 timepoint: several observations point to T0, but -do not establish the specimen conclusively. Resolving that still requires specimen -documentation; neither the alias nor this audit silently changes the timepoint. +that source receipt and the object metadata. The reviewed October metadata now +labels the Tempus tumor products T0 and records the matched normal separately as +`blood_2024-09-21`. Historical snapshots retain the `tempus-file-labels` warning. +The updated labels do not make the overlapping RNA products independent; the +specimen-provenance investigation remains open. ## Catalogue - **Samples** are what was collected: tumor biopsies and resections at time points T0 - to T3, an organoid grown from the T1 tumor, and blood draws. + to T3, organoids grown from the T1 and T2 tumors, and blood draws. - **Files** are what's in the public S3 bucket: aligned reads (BAMs), raw reads (FASTQs), variant calls, expression tables, scans and more, nearly 400,000 in all. -Each sample has BAMs and FASTQ folders. Most files belong to no sample: imaging, +Each sample lists its BAMs and FASTQ locations. A location can be a directory or +a filename prefix. Most files belong to no sample: imaging, pathology slides, analyses and references. The examples open your most recent snapshot; see [get started](index.md#get-started). @@ -57,23 +59,27 @@ data.samples.select(timepoint="T0", tissue="tumor") data.samples.select(assay="scrna-seq", platform="ont") ``` -The single-cell Oxford Nanopore samples in the 2026-09-25 snapshot: +The single-cell Oxford Nanopore samples in the 2026-10-07 snapshot: ```text 3 tumor samples -sample timepoint date where sequencing BAMs FASTQ folders --------- --------- ---------- ----- --------------------------------------- ---- ------------- -T1_tumor T1 2024-06-06 UCLA rna-seq, wes, wgs, scrna-seq (ont, 14 14 - pacbio) -T2_tumor T2 2025-01-28 UCLA rna-seq, wgs, scrna-seq (ont), cite-seq 8 7 -T3_tumor T3 2025-04-17 MSKCC scrna-seq (ont), cite-seq 2 5 +sample timepoint date where sequencing BAMs FASTQ locations +-------- --------- ---------- ----- ------------------------------------------ ---- --------------- +T1_tumor T1 2024-06-06 UCLA rna-seq, wes, wgs, scrna-seq (ont, pacbio) 12 10 +T2_tumor T2 2025-01-28 UCLA rna-seq, wgs, scrna-seq (ont), cite-seq 8 7 +T3_tumor T3 2025-04-17 MSKCC scrna-seq (ont), cite-seq 2 5 ``` One sequencing run can have several BAMs, such as a vendor's and a reprocessed one. `data.samples["T1_tumor"]` shows one sample: where it came from, each BAM with its assay, provider, size and whether you've downloaded it, each FASTQ folder with its -library, file count and size, and the calls that fetch them. +library, FASTQ-only file count and size, and the calls that fetch them. Filename +prefixes include the CeGaT, regenerated Natera and UCLA deliveries. The Tempus +`DNA/FastQ/T` and `DNA/FastQ/N` row labels resolve to the full tumor and normal +filename prefixes within their shared directory; they are not actual folders. +`sample.fastq_folders` retains these published location labels, while the displayed +locations and download commands use the resolved prefixes. ### What the fields mean @@ -86,8 +92,14 @@ library, file count and size, and the calls that fetch them. | Platform | ont | The sequencing technology, when the site says | | Provider | BostonGene | Who produced the data | -T0_tumor is the primary tumor resection on 2022-12-16, and T0_blood is blood from -the same visit. T1_tumor and T1_organoid share a time point but not a tissue. +T0_tumor is the primary tumor resection on 2022-12-16. T0_blood is a historical +grouping of normal products, not proof that all blood was drawn that day: its +current notes identify the BostonGene draw as 2023-01-27, while Natera normal +collection details remain unconfirmed. The Personalis normal is recorded separately +as blood_2024-06-11. T1_tumor and T1_organoid share a time point but not a tissue. +T2_organoid comes from the 2025-01-28 UCLA biopsy; only its WGS is assigned to the +sample because the delivered RNA contains substantial contamination from another +person. The WGS notes also flag mouse reads and required preprocessing. T3_tumor_CD45neg is a CD45-negative fraction of T3_tumor. Later blood draws are named by date, such as blood_2025-06-26, and have no time point. A snapshot's date is when you downloaded the website's metadata, not a collection date. @@ -107,10 +119,16 @@ and its corrections name the [fixes](corrections.md) that changed it. ### Where sequencing comes from A sample's sequencing combines the site's sample registry with its FASTQ table, -which records each folder's assay. The four 2026 blood draws have nothing in the -registry, but their FASTQ folders show single-cell gene expression, T-cell receptor +which records each location's assay. The eleven 2026 blood entries in the October 7 +snapshot have no assays in the registry, but their FASTQ folders show single-cell +gene expression, T-cell receptor and CITE-seq libraries. +The FASTQ table also supplies each matched file's assay and provider for file +filters. Its library-specific labels take precedence over a broader data-page +directory, such as CeGaT RNA delivered inside a WES folder; conflicting claims +about an exact file remain visible. + The site's labels map to osteosarc's names like this; using a site label as a filter gives an error that names the right one: @@ -119,6 +137,7 @@ gives an error that names the right one: | RNA, bulk RNA | rna-seq | | Bulk RNA sequencing | | WES | wes | | Whole-exome DNA sequencing | | WGS | wgs | | Whole-genome DNA sequencing | +| Panel | panel | | Targeted DNA panel (Tempus xT) | | scRNA, scRNA_GEX | scrna-seq | gene expression | Single-cell RNA sequencing | | scRNA_TCR, scRNA_TCRgd, scRNA_BCR | scrna-seq | αβ TCR, γδ TCR, BCR | T- and B-cell receptor libraries from the same cells | | scRNA_ONT | scrna-seq, platform ont | long reads | Single-cell Oxford Nanopore RNA sequencing | diff --git a/docs/testing.md b/docs/testing.md index 69b45d2..3fff607 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -60,15 +60,27 @@ On 2026-09-18 this returned 3,788 reads, and reopened the same result offline. ## Last checked -On a snapshot downloaded 2026-09-25 (UTC): +On a snapshot downloaded 2026-10-07 (UTC), using the site's bucket inventory +generated 2026-10-01: | Check | Result | | --- | --- | -| Files | 395,541, including 843 alignments and 323 VCF or BCF files | -| Variants | 181 on the variants page: 179 ready, 1 with a placeholder allele, 1 with none | -| Corrections | 35: 30 applied, 5 already fixed on the site, none stale | +| Files | 397,038 bucket objects; 397,043 catalogue entries including site tables; 913 alignments and 357 VCF or BCF files | +| Variants | 183 on the variants page: 182 ready, 1 with a placeholder allele | +| Corrections | 35: 13 applied, 22 already fixed on the site, none stale | | Unknown labels | None | -| Timeline and samples | 790 events and 21 samples | +| Timeline and samples | 807 events and 31 samples (24 blood, 5 tumor/fractions, 2 organoids) | + +The sample audit checks every registered BAM and FASTQ location, including the +eleven entries represented as filename prefixes or Tempus row aliases rather +than directories. The current tumor dates and collection sites agree with the +timepoint summary. Regression excerpts and original source receipts are in +`tests/data/sample-locations-2026-10-07.json`; older fixtures remain unchanged. +Live S3 listings matched the inventory for all 317 FASTQ locations, and HEAD +requests matched the sizes and modification dates of all 94 sample BAMs. One +published FASTQ-table count differs: `rna-seq/tempus/TL-24-ALMY2X4KMV/RNA` lists +two compressed FASTQs (1,238,829,178 bytes), but also contains `2022_R1.fastq`. +The inventory and sample view retain all three files (4,470,300,364 bytes). The 2026-09-18 snapshot, from before the site renamed five variants and merged two USH2A entries, still applies all 35 corrections to its 182 entries. Every allele diff --git a/osteosarc/__init__.py b/osteosarc/__init__.py index 6ee3932..528df7f 100644 --- a/osteosarc/__init__.py +++ b/osteosarc/__init__.py @@ -47,9 +47,11 @@ ) from .read_receipts import read_receipt_files # noqa: F401 from .reads import ( # noqa: F401 + ReadCount, ReadFilter, ReadSubset, assembly_from_header, + count_reads, extract_reads, inspect_alignment, ) diff --git a/osteosarc/catalog.py b/osteosarc/catalog.py index 3e9e121..262a628 100644 --- a/osteosarc/catalog.py +++ b/osteosarc/catalog.py @@ -10,6 +10,7 @@ from .cache import stable_id from .curation import ( ASSAYS, + FASTQ_PREFIX_ALIASES, label_assay, label_claims, normalize_provider, @@ -109,7 +110,7 @@ def object_key(value, base=BUCKET): return value -def build_files(listing, bams, metadata, vafs, path_claims=(), *, tables=None): +def build_files(listing, bams, metadata, vafs, path_claims=(), *, tables=None, fastqs=()): """Retain every listed object, enriching exact paths before basename matches. Basename joins are used only when unique among alignment objects. Path @@ -159,13 +160,25 @@ def build_files(listing, bams, metadata, vafs, path_claims=(), *, tables=None): by_path = defaultdict(list) for prefix, claim in path_claims: by_path[prefix.rstrip("/")].append(claim) + fastq_rows = [(object_key(row["s3_folder"], base), row) for row in fastqs if row.get("s3_folder")] files = [] for key, object_metadata in objects.items(): kind, format = file_type(key) info = dict(extra.get(key, {})) records = list(claims.get(key, ())) + if kind == "reads": + for location, row in fastq_rows: + if matches_fastq(key, location): + assay, platform = ASSAYS.get(row.get("assay"), (None, None)) + by_path[key].append(SampleClaim( + "fastqs", row.get("display_name", ""), normalize_timepoint(row.get("timepoint")), + row.get("sample_date") or None, assay, platform, + normalize_tissue(row.get("tissue")), normalize_provider(row.get("provider")))) + info.setdefault("fastq_rows", []).append(dict(row)) # The most specific data-page path wins: a file's own row overrides the # row for its directory, which can also hold other assays' files. + # A matched FASTQ library row likewise names the actual read files and + # overrides a broader directory's assay (e.g. CeGaT RNA filed under WES). parts = key.split("/") for depth in range(len(parts), 0, -1): if "/".join(parts[:depth]) in by_path: @@ -232,7 +245,9 @@ def parse_data_paths(html): result = [] for context, values in data_page_rows(html): title = context.lower() - assay = ("wgs" if "whole genome" in title or re.search(r"\bwgs\b", title) else + assay = (None if "variant calling" in title else + "panel" if "panel" in values.get("Provider", "").lower() else + "wgs" if "whole genome" in title or re.search(r"\bwgs\b", title) else "wes" if "whole exome" in title or re.search(r"\bwes\b", title) else "scrna-seq" if any(x in title for x in ("single cell", "single-cell")) else "rna-seq" if "bulk rna" in title else None) @@ -247,3 +262,26 @@ def parse_data_paths(html): assay=assay, platform=platform, tissue=tissue, provider=normalize_provider(values.get("Provider"))))) return tuple(result) + + +def fastq_prefix(location): + """Effective bucket prefix; retain the published location separately.""" + location = location.rstrip("/") + return FASTQ_PREFIX_ALIASES.get(location, location) + + +def matches_fastq(key, location): + """Match a FASTQ directory, exact file or filename prefix, never sidecars. + + A filename prefix ends at a separator so P116686_1 cannot claim P116686_10. + This interprets published locations only; it does not infer sample identity. + """ + if file_type(key) != ("reads", "fastq"): + return False + prefix = fastq_prefix(location) + if not prefix or not key.startswith(prefix): + return False + tail = key[len(prefix):] + if not tail or tail.startswith("/"): + return True + return "/" not in tail and (prefix.endswith(("_", "-", ".")) or tail[0] in "_.-") diff --git a/osteosarc/curation.py b/osteosarc/curation.py index 022f341..7ef203a 100644 --- a/osteosarc/curation.py +++ b/osteosarc/curation.py @@ -56,10 +56,10 @@ # libraries from the same single-cell captures, and bulk RNA. "scRNA_GEX": ("scrna-seq", None), "scRNA_TCR": ("scrna-seq", None), "scRNA_TCRgd": ("scrna-seq", None), "scRNA_BCR": ("scrna-seq", None), - "bulk RNA": ("rna-seq", None)} + "bulk RNA": ("rna-seq", None), "Panel": ("panel", None)} #: Filter values for file metadata, in display order. -ASSAY_NAMES = ("rna-seq", "wes", "wgs", "scrna-seq", "cite-seq") +ASSAY_NAMES = ("rna-seq", "wes", "wgs", "scrna-seq", "cite-seq", "panel") PLATFORM_NAMES = ("illumina", "ont", "pacbio") TISSUE_NAMES = ("tumor", "blood", "organoid") @@ -81,6 +81,15 @@ def sequencing_pairs(labels): PROVIDERS = ("BostonGene", "CeGaT", "Hudson Lab", "Natera", "Personalis", "Tempus", "UCLA", "UCSF") PROVIDER_ALIASES = {"Boston Gene": "BostonGene"} +# The consolidated FASTQ table uses these two specimen labels inside a shared +# directory. Its notes explicitly describe /T and /N as row keys, not folders. +# Keep the source values; use the full accession and tissue token for discovery. +FASTQ_PREFIX_ALIASES = { + f"vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/{tissue}": + f"vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_{tissue}_" + for tissue in ("T", "N") +} + #: "T1-organoid" is timepoint T1; the organoid specimen is recorded as tissue. TIMEPOINT_ALIASES = {"T1-organoid": "T1"} @@ -89,7 +98,10 @@ def sequencing_pairs(labels): #: A composite "Tumor + Normal" row stays unresolved rather than guessed. TISSUES = {"tumor": "tumor", "blood": "blood", "normal": "blood", "normal (blood)": "blood", "blood normal": "blood", "blood/normal": "blood", "organoid": "organoid", - "tumor + normal": None} + "tumor + normal": None, "blood normal (drawn 2024-06-11)": "blood", + "normal (blood, drawn 2024-06-11)": "blood", "normal (blood, drawn 2024-09-21)": "blood", + "organoid (read 1)": "organoid", "organoid (read 2)": "organoid", + "tumor + blood (drawn 2024-09-21)": None} #: Detection and vaccine keys present when the curation below was written. #: New keys are not errors, but they are reported as possible upstream drift. @@ -119,6 +131,10 @@ def check_filter(name, value, present=frozenset): def normalize_provider(value): + # Data-page cells append delivery or processing notes to a known provider. + name = (value or "").split(" (", 1)[0] + if name in PROVIDERS or name in PROVIDER_ALIASES: + value = name value = PROVIDER_ALIASES.get(value, value) return value or None @@ -963,10 +979,38 @@ def _stale_viewer_label(library, old, new, note): Correction( "natera-alleles-unavailable", "COL3A1 (splice) and OTUD4 (p.Ala153del) come from the Natera 2022 report, which is not " - "public. An independent public Tempus call now resolves COL3A1; OTUD4 remains unavailable.", + "public. Independent public calls resolve COL3A1 and, for the reviewed 2026-10-05 " + "layout, a matching OTUD4 allele. These calls do not verify the original Natera report " + "or establish somatic status.", (Change("variant_index", {"id": "COL3A1-Splice"}), Change("variant_index", {"id": "OTUD4-p_A153del"})), evidence=(_SITE_REPO + "scripts/variants/source_data/SS%20neoantigen%20_%20mutations%20-%20Mutations.tsv", _BUCKET + "vendor/natera/manifest.tsv"), verified="2026-09-18"), ) + + +def _reviewed_revisions(corrections): + """Accept evidence-reviewed layouts while retaining all historical drift guards.""" + review = json.loads(files("osteosarc").joinpath("data/curation-revisions.json").read_text()) + result = [] + for correction in corrections: + revision = review["corrections"].get(correction.id) + if revision is not None: + versions = [] + for version in revision["versions"]: + changes = [] + for specification in version: + match = {key: glob(value["glob"]) if isinstance(value, dict) else value + for key, value in specification["match"].items()} + changes.append(Change(**dict(specification, match=match))) + versions.append(tuple(changes)) + correction = replace( + correction, alternatives=(*correction.alternatives, *versions), + summary=correction.summary + revision["summary_suffix"], verified=review["verified"], + evidence=tuple(dict.fromkeys((*correction.evidence, *review["evidence"])))) + result.append(correction) + return tuple(result) + + +CORRECTIONS = _reviewed_revisions(CORRECTIONS) diff --git a/osteosarc/data/curation-revisions.json b/osteosarc/data/curation-revisions.json new file mode 100644 index 0000000..992b2a9 --- /dev/null +++ b/osteosarc/data/curation-revisions.json @@ -0,0 +1,1118 @@ +{ + "verified": "2026-10-05", + "corrections": { + "tempus-grch37-counts": { + "versions": [ + [ + { + "source": "vafs", + "match": { + "bam_file": { + "glob": "*TL-24-5GQLV9WSXQ*" + } + }, + "expect": {}, + "set": {}, + "absent": true + }, + { + "source": "bams", + "match": { + "url": { + "glob": "*TL-24-5GQLV9WSXQ*" + } + }, + "expect": {}, + "set": {}, + "absent": true + }, + { + "source": "bam_metadata", + "match": { + "s3_path": "vendor/tempus/TL-24-5GQLV9WSXQ/DNA/TL-24-5GQLV9WSXQ_T.sorted.bam" + }, + "expect": { + "provider": "Tempus", + "assay": "Panel", + "timepoint": "T0", + "tissue": "Tumor" + }, + "set": {}, + "absent": false + }, + { + "source": "vafs", + "match": { + "variant_id": "PDZRN4-chr12-41572809" + }, + "expect": {}, + "set": {}, + "absent": false + } + ] + ], + "summary_suffix": " The 2026-10-05 metadata identifies this product as a T0 targeted panel; its invalid count rows remain removed." + }, + "gene-symbol-TRMO": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "TMRO-chr9-97910412" + }, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "TMRO-chr9-97910412" + }, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "TMRO-chr9-97910412" + }, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "TRMO-chr9-97910412" + }, + "expect": { + "gene": "TRMO", + "chr": "chr9", + "pos": 97910412, + "ref": "C", + "alt": "T", + "genomic_location": "chr9:97910412" + }, + "set": { + "gene": "TRMO" + } + }, + { + "source": "variant_index", + "match": { + "id": "TRMO-chr9-97910412" + }, + "expect": { + "gene": "TRMO", + "location": "chr9:97910412" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "TRMO-chr9-97910412" + }, + "expect": { + "gene": "TRMO", + "chrom": "chr9", + "pos": "97910412", + "ref": "C", + "alt": "T" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sources consistently rename the ID to TRMO-chr9-97910412." + }, + "allele-DCHS2-chr4-154322488": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "DCHS2-chr4-154322488" + }, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "DCHS2-chr4-154322488" + }, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "DCHS2-chr4-154322488" + }, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "DCHS2-chr4-154323273" + }, + "expect": { + "gene": "DCHS2", + "chr": "chr4", + "pos": 154323273, + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "genomic_location": "chr4:154323273" + }, + "set": { + "chr": "chr4", + "pos": 154323273, + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA" + } + }, + { + "source": "variant_index", + "match": { + "id": "DCHS2-chr4-154323273" + }, + "expect": { + "gene": "DCHS2", + "location": "chr4:154323273" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "DCHS2-chr4-154323273" + }, + "expect": { + "gene": "DCHS2", + "chrom": "chr4", + "pos": "154323273", + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "status": "unsupported", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "" + }, + "set": { + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sources publish the verified literal allele and mark every count unsupported and missing; those missing values remain missing." + }, + "allele-GAPVD1-chr9-125299105": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "GAPVD1-chr9-125299105" + }, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "GAPVD1-chr9-125299105" + }, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "GAPVD1-chr9-125299105" + }, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "GAPVD1-chr9-125301980" + }, + "expect": { + "gene": "GAPVD1", + "chr": "chr9", + "pos": 125301980, + "ref": "TAGTGC", + "alt": "ATTGG", + "genomic_location": "chr9:125301980" + }, + "set": { + "chr": "chr9", + "pos": 125301980, + "ref": "TAGTGC", + "alt": "ATTGG" + } + }, + { + "source": "variant_index", + "match": { + "id": "GAPVD1-chr9-125301980" + }, + "expect": { + "gene": "GAPVD1", + "location": "chr9:125301980" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "GAPVD1-chr9-125301980" + }, + "expect": { + "gene": "GAPVD1", + "chrom": "chr9", + "pos": "125301980", + "ref": "TAGTGC", + "alt": "ATTGG", + "status": "unsupported", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "" + }, + "set": { + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sources publish the verified literal allele and mark every count unsupported and missing; those missing values remain missing." + }, + "allele-MAP2-chr2-209694768": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "MAP2-chr2-209694768" + }, + "expect": { + "gene": "MAP2", + "chr": "chr2", + "pos": 209694768, + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG", + "genomic_location": "chr2:209694768", + "genomic_change_on_cdna": "c.2599_2630delinsAGGG" + }, + "set": { + "chr": "chr2", + "pos": 209694768, + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "MAP2-chr2-209694768" + }, + "expect": { + "chrom": "chr2", + "pos": "209694768", + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sources publish this same complex allele as delins; its source type and updated counts are preserved." + }, + "map2-split-representations": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "MAP2-chr2-209694769" + }, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "MAP2-chr2-209694769" + }, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "MAP2-chr2-209694769" + }, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "MAP2-chr2-209694772" + }, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "MAP2-chr2-209694772" + }, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "MAP2-chr2-209694772" + }, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "MAP2-chr2-209694768" + }, + "expect": { + "gene": "MAP2", + "chr": "chr2", + "pos": 209694768, + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG", + "genomic_location": "chr2:209694768" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sources remove both split entries and retain the verified complex event." + }, + "fam157a-withdrawn-protein": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "FAM157A-p_W70_Q71ins_14" + }, + "expect": { + "gene": "FAM157A", + "chr": "chr3", + "pos": 198153259, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "genomic_location": "chr3:198153259", + "refseq_id": "NM_001145248", + "note": "Tempus pindel GRCh37 3:197880130 G>G+42 (5/39 tumor reads, 0/30 normal), lifted to GRCh38 chr3:198153259. The site is the germline-polymorphic CAG repeat rs56683636, where the patient's normal DNA is heterozygous for 6-bp and 9-bp deletions, so this is a low-confidence Tempus-only call; the allele is not countable by the site pileup. Protein caveat: NCBI has suppressed NM_001145248.1 (support for the transcript but not for the protein) and now annotates FAM157A as non-coding (NR_146164.1, whose exon 1 contains this insertion), so p.W70_Q71ins(14) derives from a retired protein model and is not supported by current annotation." + }, + "set": { + "note": "Tempus pindel GRCh37 3:197880130 G>G+42 (5/39 tumor reads, 0/30 normal), lifted to GRCh38 chr3:198153259. The site is the germline-polymorphic CAG repeat rs56683636, where the patient's normal DNA is heterozygous for 6-bp and 9-bp deletions, so this is a low-confidence Tempus-only call; the allele is not countable by the site pileup. Protein caveat: NCBI has suppressed NM_001145248.1 (support for the transcript but not for the protein) and now annotates FAM157A as non-coding (NR_146164.1, whose exon 1 contains this insertion), so p.W70_Q71ins(14) derives from a retired protein model and is not supported by current annotation." + } + }, + { + "source": "variant_index", + "match": { + "id": "FAM157A-p_W70_Q71ins_14" + } + } + ] + ], + "summary_suffix": " The expanded 2026-10-05 note retains the withdrawn-protein caveat and identifies the low-confidence repeat call." + }, + "ush2a-transposed-duplicate": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "USH2A-chr1-215560752" + }, + "expect": {}, + "set": {}, + "absent": true + }, + { + "source": "variant_index", + "match": { + "id": "USH2A-chr1-215560752" + }, + "expect": {}, + "set": {}, + "absent": true + }, + { + "source": "vafs", + "match": { + "variant_id": "USH2A-chr1-215560752" + }, + "expect": {}, + "set": {}, + "absent": true + }, + { + "source": "source_variants", + "match": { + "id": "USH2A-chr1-215650752" + }, + "expect": { + "gene": "USH2A", + "chr": "chr1", + "pos": 215650752, + "ref": "C", + "alt": "A", + "genomic_change_on_cdna": "c.14183G>T", + "refseq_id": "NM_206933", + "protein_change": "p.Cys4728Phe", + "genomic_location": "chr1:215650752", + "genomic_ref_context": "ACATGGTGCAGAACC" + }, + "set": { + "genomic_location": "chr1:215650752", + "genomic_ref_context": "ACATGGTGCAGAACC", + "upstream_merge": { + "retired_id": "USH2A-chr1-215560752", + "retained_id": "USH2A-chr1-215650752", + "evidence": "https://gitlab.com/slowkow/osteosarc.com/-/commit/bc138897b77323f595acffa46cbfce5c45720a1d", + "original_source_identity": "confirmed", + "identity_basis": "chr1:215560752 lies outside USH2A (chr1:215621576-216423448), so it cannot encode p.Cys4728Phe" + } + }, + "absent": false + }, + { + "source": "variant_index", + "match": { + "id": "USH2A-chr1-215650752" + }, + "expect": { + "gene": "USH2A", + "location": "chr1:215650752", + "protein_label": "p.Cys4728Phe" + }, + "set": {}, + "absent": false + }, + { + "source": "vafs", + "match": { + "variant_id": "USH2A-chr1-215650752" + }, + "expect": { + "gene": "USH2A", + "chrom": "chr1", + "pos": "215650752", + "ref": "C", + "alt": "A" + }, + "set": {}, + "absent": false + } + ] + ], + "summary_suffix": " The 2026-10-05 source also fixes the retained location and context; the retired-ID relationship is still attached." + }, + "allele-FAM157A-p_W70_Q71ins_14": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "FAM157A-p_W70_Q71ins_14" + }, + "expect": { + "gene": "FAM157A", + "chr": "chr3", + "pos": 198153259, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "genomic_change_on_cdna": "c.211_212ins(42)", + "refseq_id": "NM_001145248", + "protein_change": "p.Trp70_Gln71ins(14)", + "genomic_location": "chr3:198153259" + }, + "set": { + "allele_resolution": { + "status": "resolved", + "source_allele": { + "assembly": "GRCh37", + "chrom": "chr3", + "pos": 197880130, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA" + }, + "allele": { + "assembly": "GRCh38", + "chrom": "chr3", + "pos": 198153259, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA" + }, + "source": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV.soma.pindel.vcf", + "sha256": "f3b39c22454ed0269b9e163e1f09f406bc2c1a57c48735233915901e6dc58090" + }, + "references": [ + { + "accession": "NC_000003.11", + "start": 197879630, + "end": 197880630, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000003.11&seq_start=197879630&seq_stop=197880630&rettype=fasta&retmode=text", + "sha256": "f308a720abf846c7be3fe9f9e433eafd40d3b622306c063750f05998f8b56353" + }, + { + "accession": "NC_000003.12", + "start": 198152759, + "end": 198153759, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000003.12&seq_start=198152759&seq_stop=198153759&rettype=fasta&retmode=text", + "sha256": "bf700b25dc5b4cdfbf0f4bce7e09aae159926ae08d5cabad734dc081e6305115" + } + ], + "mapping": { + "url": "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz", + "sha256": "5c0598e500ceb5a78c73086929e8ef993aec309bcafb595139b53d440b125a1d", + "coordinates": "0-based half-open", + "blocks": [ + { + "chain_id": "3", + "source": [ + "chr3", + 197879629, + 197880630 + ], + "target": [ + "chr3", + 198152758, + 198153759 + ], + "strand": "+" + } + ] + }, + "normalization": { + "representation": "leftmost anchored allele in the checked reference window", + "equivalent_positions_grch38": [ + 198153259, + 198153260, + 198153261 + ], + "identical_reference_windows": true + }, + "transcript": "NM_001145248.1", + "source_hgvs_c": "c.211_212insGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCAGC", + "summary": "The public Tempus Pindel call supplies the 42-base insertion. Its REF and flanking sequence match both assemblies.", + "protein_interpretation": { + "status": "withdrawn_protein_model", + "transcript": "NM_001145248.1", + "current_transcript": "NR_146164.1", + "source": { + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=gene&id=728262&retmode=xml", + "sha256": "a6ed3ce8173d5fafe94a01960d1b35bf2eab3f59ccaf4413c0a0665873842378" + }, + "reason": "NCBI supports the transcript but not the protein; FAM157A is now annotated as ncRNA." + }, + "caveats": [ + "Genomic readiness does not establish a protein-coding insertion or a neoantigen.", + "No read-count rows are published for this catalogue entry." + ] + }, + "chr": "chr3", + "pos": 198153259, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "variant_type": "insertion", + "genomic_location": "chr3:198153259" + } + }, + { + "source": "variant_index", + "match": { + "id": "FAM157A-p_W70_Q71ins_14" + }, + "expect": { + "gene": "FAM157A", + "location": "chr3:198153259", + "protein_label": "p.Trp70_Gln71ins(14)" + }, + "set": { + "location": "chr3:198153259" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "FAM157A-p_W70_Q71ins_14" + }, + "expect": { + "chrom": "chr3", + "pos": "198153259", + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 source supplies this verified allele; the reviewed acquisition and reference evidence remains attached without replacing its count rows." + }, + "allele-COL3A1-Splice": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "COL3A1-Splice" + }, + "expect": { + "gene": "COL3A1", + "chr": "chr2", + "pos": 189010889, + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C", + "genomic_change_on_cdna": "c.4254+1_4255-1del", + "refseq_id": "NM_000090", + "protein_change": "Splice", + "genomic_location": "chr2:189010889" + }, + "set": { + "allele_resolution": { + "status": "resolved", + "source_allele": { + "assembly": "GRCh37", + "chrom": "chr2", + "pos": 189875615, + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C" + }, + "allele": { + "assembly": "GRCh38", + "chrom": "chr2", + "pos": 189010889, + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C" + }, + "source": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV.soma.pindel.vcf", + "sha256": "f3b39c22454ed0269b9e163e1f09f406bc2c1a57c48735233915901e6dc58090" + }, + "references": [ + { + "accession": "NC_000002.11", + "start": 189875115, + "end": 189876852, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000002.11&seq_start=189875115&seq_stop=189876852&rettype=fasta&retmode=text", + "sha256": "66ccf5a5c3ff62c3f01a20c4de6e92a95ed636b65aae220972a7e8934f5a50a6" + }, + { + "accession": "NC_000002.12", + "start": 189010389, + "end": 189012126, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000002.12&seq_start=189010389&seq_stop=189012126&rettype=fasta&retmode=text", + "sha256": "ba0cca8c5f8ff532111a5f1cf324a1acb722c8162b1139f3fbb4545ceaa30472" + } + ], + "mapping": { + "url": "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz", + "sha256": "5c0598e500ceb5a78c73086929e8ef993aec309bcafb595139b53d440b125a1d", + "coordinates": "0-based half-open", + "blocks": [ + { + "chain_id": "1", + "source": [ + "chr2", + 189875114, + 189876852 + ], + "target": [ + "chr2", + 189010388, + 189012126 + ], + "strand": "+" + } + ] + }, + "normalization": { + "representation": "leftmost anchored allele in the checked reference window", + "equivalent_positions_grch38": [ + 189010889, + 189010890 + ], + "identical_reference_windows": true + }, + "transcript": "NM_000090.3", + "source_hgvs_c": "c.4254+1_4255-1del", + "summary": "The public Tempus Pindel call matches the catalogue transcript and c.4254+1_4255-1del annotation: a 737-base intronic deletion. Its full REF and flanking sequence match both assemblies.", + "caveats": [ + "The original Natera report is not public; the allele is supported by the independent public Tempus call.", + "No read-count rows are published for this catalogue entry." + ] + }, + "chr": "chr2", + "pos": 189010889, + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C", + "variant_type": "deletion", + "genomic_location": "chr2:189010889" + } + }, + { + "source": "variant_index", + "match": { + "id": "COL3A1-Splice" + }, + "expect": { + "gene": "COL3A1", + "location": "chr2:189010889", + "protein_label": "Splice" + }, + "set": { + "location": "chr2:189010889" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "COL3A1-Splice" + }, + "expect": { + "chrom": "chr2", + "pos": "189010889", + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 source supplies this verified allele; the reviewed acquisition and reference evidence remains attached without replacing its count rows." + }, + "otud4-source-unavailable": { + "versions": [ + [ + { + "source": "source_variants", + "match": { + "id": "OTUD4-p_A153del" + }, + "expect": { + "gene": "OTUD4", + "chr": "chr4", + "pos": 145155970, + "ref": "TCAG", + "alt": "T", + "genomic_change_on_cdna": "c.458_460del", + "refseq_id": "NM_001102653", + "protein_change": "p.Ala153del", + "genomic_location": "chr4:145155970" + }, + "set": { + "allele_resolution": { + "status": "resolved", + "summary": "A public CeGaT normal-DNA call supplies the matching literal OTUD4 allele and transcript annotation. This does not verify the original Natera call or establish somatic status.", + "allele": { + "assembly": "GRCh38", + "chrom": "chr4", + "pos": 145155970, + "ref": "TCAG", + "alt": "T" + }, + "source_allele": { + "assembly": "GRCh37", + "chrom": "chr4", + "pos": 146077122, + "ref": "TCAG", + "alt": "T" + }, + "source": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/cegat/P116686_1_S000048/P116686_1_indels.vcf", + "sha256": "4e37a807db95fdc82b36e0512f8bc3f58a021bdd8268d3e59fde5fd9a8d29683" + }, + "annotation_source": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/cegat/P116686_1_S000048/P116686_1_indels.annotated.tsv", + "sha256": "18650e9b0cd000a8c7b58b3a562f7a621a8d61ff9c9deb389428ba542d99215e" + }, + "source_filter": "FP/HET", + "source_counts": { + "alt": 7, + "ref": 316, + "depth": 323 + }, + "transcript": "NM_001102653.1", + "source_hgvs_c": "c.458_460delCTG", + "source_hgvs_p": "p.A153del", + "references": [ + { + "accession": "NC_000004.11", + "start": 146076622, + "end": 146077622, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000004.11&seq_start=146076622&seq_stop=146077622&rettype=fasta&retmode=text", + "sha256": "564dd3888f5ae4ed9b5248cc8ab41c52d18b1e90c12cbe15f2999698a923d999" + }, + { + "accession": "NC_000004.12", + "start": 145155470, + "end": 145156470, + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000004.12&seq_start=145155470&seq_stop=145156470&rettype=fasta&retmode=text", + "sha256": "7563c0185bb92d5e129b4c679741b0a686d0b7d61e49ae87124edbde96be8e78" + } + ], + "mapping": { + "url": "https://rest.ensembl.org/map/human/GRCh37/4:146077122..146077125:1/GRCh38?content-type=application/json", + "sha256": "e72fb653a74d8a092177dc6ad85135b40028e9187b7e90e0888f378ffc0b5087", + "mappings": [ + { + "original": { + "end": 146077125, + "strand": 1, + "start": 146077122, + "coord_system": "chromosome", + "assembly": "GRCh37", + "seq_region_name": "4" + }, + "mapped": { + "start": 145155970, + "end": 145155973, + "strand": 1, + "assembly": "GRCh38", + "seq_region_name": "4", + "coord_system": "chromosome" + } + } + ] + }, + "original_natera_report": "unavailable", + "somatic_status": "not_established", + "source_tissue": "blood", + "source_metadata": { + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/bam-metadata-consolidated.tsv", + "sha256": "cb620face7a055283818eea668a18695d12dc280e0ca7666cb637d37938da2ef", + "display_name": "T0 CeGaT Blood WES 2022-12 (P116686_1 vendor)", + "source_bam": "vendor/cegat/P116686_1_S000048/P116686_1.bam" + } + } + } + }, + { + "source": "variant_index", + "match": { + "id": "OTUD4-p_A153del" + }, + "expect": { + "gene": "OTUD4", + "location": "chr4:145155970", + "protein_label": "p.Ala153del" + } + }, + { + "source": "vafs", + "match": { + "variant_id": "OTUD4-p_A153del" + }, + "expect": { + "chrom": "chr4", + "pos": "145155970", + "ref": "TCAG", + "alt": "T" + } + } + ] + ], + "summary_suffix": " On 2026-10-05, a matching public CeGaT normal-DNA call (FP/HET, 7/323) and RefSeq/assembly mapping verify the newly published literal allele. The original Natera report remains unavailable and somatic status is not established." + }, + "specimen-T1-site": { + "versions": [ + [ + { + "source": "specimens", + "match": { + "sample_id": "T1_tumor" + }, + "expect": { + "collection_site": "UCLA", + "tissue_source": "Recurrent tumor biopsy (CT-guided, T4-T5)" + }, + "set": { + "collection_site": "UCLA", + "tissue_source": "Recurrent tumor biopsy (CT-guided, T4-T5)" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 registry uses the corrected site and procedure, with additional anatomical detail; the published dates are preserved." + }, + "specimen-T3-site": { + "versions": [ + [ + { + "source": "specimens", + "match": { + "sample_id": "T3_tumor" + }, + "expect": { + "collection_site": "MSKCC", + "tissue_source": "Tumor resection (T4-T5)" + }, + "set": { + "collection_site": "MSKCC", + "tissue_source": "Tumor resection (T4-T5)" + } + }, + { + "source": "specimens", + "match": { + "sample_id": "T3_tumor_CD45neg" + }, + "expect": { + "collection_site": "MSKCC" + }, + "set": { + "collection_site": "MSKCC" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 registry uses the corrected site and procedure, with additional anatomical detail; the published dates are preserved." + }, + "tempus-file-labels": { + "versions": [ + [ + { + "source": "bam_metadata", + "match": { + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV_T.sorted.bam" + }, + "expect": { + "provider": "Tempus", + "assay": "WES", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "display_name": "T0 Tempus Tumor WES 2022-12 (TL-24-ALMY2X4KMV vendor)" + }, + "set": { + "timepoint": "T0" + } + }, + { + "source": "bam_metadata", + "match": { + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV_N.sorted.bam" + }, + "expect": { + "provider": "Tempus", + "assay": "WES", + "tissue": "Blood", + "timepoint": "", + "sample_date": "2024-09-21", + "display_name": "Tempus Blood WES 2024-09-21 (TL-24-ALMY2X4KMV vendor)" + }, + "set": { + "timepoint": "" + } + }, + { + "source": "bam_metadata", + "match": { + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/RNA/TL-24-ALMY2X4KMV_T_sorted.bam" + }, + "expect": { + "provider": "Tempus", + "assay": "RNA", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "display_name": "T0 Tempus Tumor RNA 2022-12 (TL-24-ALMY2X4KMV vendor)" + }, + "set": { + "timepoint": "T0" + } + }, + { + "source": "bam_metadata", + "match": { + "s3_path": "vendor/tempus/TL-24-KCVBE1UI1P/RNA/TL-24-KCVBE1UI1P_T_sorted.bam" + }, + "expect": { + "provider": "Tempus", + "assay": "RNA", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "display_name": "T0 Tempus Tumor RNA 2022-12 (TL-24-KCVBE1UI1P vendor)" + }, + "set": { + "timepoint": "T0" + } + }, + { + "source": "bam_metadata", + "match": { + "s3_path": { + "glob": "vendor/tempus/TL-24-ALMY2X4KMV/*" + }, + "tissue": "Tumor" + }, + "expect": { + "timepoint": "T0" + } + }, + { + "source": "bam_metadata", + "match": { + "s3_path": { + "glob": "vendor/tempus/TL-24-KCVBE1UI1P/*" + }, + "tissue": "Tumor" + }, + "expect": { + "timepoint": "T0" + } + } + ] + ], + "summary_suffix": " The reviewed 2026-10-05 metadata now labels these tumor products T0 and leaves the later normal draw without a tumor timepoint. These published labels are retained; the overlapping RNA products still are not independent evidence." + }, + "reyagel-end-date": { + "versions": [ + [ + { + "source": "events_sheet", + "match": { + "Title": "ReyaGel", + "Start date": "06/15/2026" + }, + "expect": { + "End date": "07/14/2026" + }, + "set": { + "End date": "07/14/2026" + } + }, + { + "source": "events", + "match": { + "title": "ReyaGel", + "date": "2026-06-15" + }, + "expect": { + "end_date": "2026-07-14" + } + } + ] + ], + "summary_suffix": " The 2026-10-05 sheet explicitly supplies the year and the published event includes 2026-07-14." + } + }, + "evidence": [ + "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/bam-metadata-consolidated.tsv", + "https://osteosarc.com/data/events.json", + "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/timeline/timeline.csv", + "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/src/data/variants.json", + "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/samples-consolidated.tsv", + "https://osteosarc.com/variants/variant_vafs_long.tsv", + "https://osteosarc.com/variants/" + ] +} diff --git a/osteosarc/dataset.py b/osteosarc/dataset.py index 4dd62d6..70f84c2 100644 --- a/osteosarc/dataset.py +++ b/osteosarc/dataset.py @@ -22,6 +22,7 @@ TIMELINE_SOURCES, build_files, data_page_rows, + matches_fastq, object_key, parse_data_paths, ) @@ -493,6 +494,12 @@ def _tag_samples(self, files): parts = file.key.split("/") for depth in range(1, len(parts)): found.extend(folders.get("/".join(parts[:depth]), ())) + if file.format == "fastq": + locations = [folder for folder in folders if matches_fastq(file.key, folder)] + for location in locations: + found.extend(folders[location]) + if locations: + file.metadata["fastq_locations"] = tuple(locations) if found: file.metadata["samples"] = tuple(dict.fromkeys(found)) @@ -619,7 +626,8 @@ def _build_files(self): tables = {name: (self.manifest["sources"].get(name, {}).get("url", url), format) for name, (url, format) in TABLE_SOURCES.items()} files = build_files(listing, bams, metadata, self.vafs, - parse_data_paths(read_text(self.source_path("data_page"))), tables=tables) + parse_data_paths(read_text(self.source_path("data_page"))), tables=tables, + fastqs=self.curation.records("fastqs")[0] or ()) touched = {} base = listing.get("download_base", BUCKET) for i, ids in bucket_touched.items(): @@ -971,6 +979,31 @@ def make_bundle(self, to, *, variants=(), svs=(), files=(), caps=None, unplaced_ size_budget=size_budget, header_policy=header_policy, log=log) return Path(to) + def _read_index(self, file, options): + """Use the same pinned/local index for extraction and count-only queries.""" + from .reads import require_samtools + if options.get("index") is None and file.index_urls: + offline = Cache(self.cache.root, offline=True, timeout=self.cache.timeout) + try: + index_path = self._download(file.index_urls[0], cache=offline) + except OfflineError: + if self.cache.offline: + raise + require_samtools(fetch_pairs=options.get("fetch_pairs", False), + unplaced_mates=options.get("unplaced_mates", True), filters=options.get("filters")) + index_path = self.download(file.index_urls[0]) + options["index"] = str(index_path) + + def count_reads(self, file, regions, **kwargs): + """Count bounded indexed regions without retaining BAMs; see count_reads.""" + from .reads import count_reads + regions = tuple(regions) + if not regions or not all(isinstance(r, Region) for r in regions): + raise CoordinateError("Provide a nonempty sequence of Region objects") + file = file if isinstance(file, File) else self.file(file) + self._read_index(file, kwargs) + return count_reads(file, regions, cache=self.cache, snapshot_id=self.id, **kwargs) + def extract_reads(self, file, regions=None, *, variants=None, padding=0, to=None, name=None, **kwargs): """Stream just the reads in some regions, or around variants, into a small indexed BAM. @@ -981,7 +1014,7 @@ def extract_reads(self, file, regions=None, *, variants=None, padding=0, to=None asked for, such as BG003082.MAP2-chr2-209694768.bam. Other options: see osteosarc.extract_reads. """ - from .reads import ReadSubset, extract_reads, require_samtools + from .reads import ReadSubset, extract_reads if variants is not None: if regions is not None: raise ValueError("Supply either regions or variants, not both") @@ -993,19 +1026,7 @@ def extract_reads(self, file, regions=None, *, variants=None, padding=0, to=None if not regions or not all(isinstance(r, Region) for r in regions): raise CoordinateError("Provide a nonempty sequence of regions or ready variants") file = file if isinstance(file, File) else self.file(file) - if kwargs.get("index") is None and file.index_urls: - # Try the pinned/local index first so cached reads work without - # SAMtools. Check capabilities before an index needs downloading. - offline = Cache(self.cache.root, offline=True, timeout=self.cache.timeout) - try: - index_path = self._download(file.index_urls[0], cache=offline) - except OfflineError: - if self.cache.offline: - raise - require_samtools(fetch_pairs=kwargs.get("fetch_pairs", False), - unplaced_mates=kwargs.get("unplaced_mates", True), filters=kwargs.get("filters")) - index_path = self.download(file.index_urls[0]) - kwargs["index"] = str(index_path) + self._read_index(file, kwargs) subset = extract_reads(file, regions, cache=self.cache, snapshot_id=self.id, **kwargs) if to is None: return subset diff --git a/osteosarc/models.py b/osteosarc/models.py index 4ebf3ec..1e8f2d4 100644 --- a/osteosarc/models.py +++ b/osteosarc/models.py @@ -303,7 +303,8 @@ class Sample: sequencing lists (assay, platform) pairs from the site's sample registry and from the sample's FASTQ folders. bams are file keys; fastq_folders are - bucket folders. missing_bams are BAMs the registry names but the bucket + published bucket folders or filename prefixes (including source aliases). + missing_bams are BAMs the registry names but the bucket doesn't have. """ diff --git a/osteosarc/reads.py b/osteosarc/reads.py index 0508392..c067f89 100644 --- a/osteosarc/reads.py +++ b/osteosarc/reads.py @@ -11,6 +11,7 @@ import tempfile import time from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext from dataclasses import asdict, dataclass from pathlib import Path from threading import Event, Timer @@ -151,6 +152,14 @@ def open(self): return pysam.AlignmentFile(str(self.path), index_filename=str(self.index_path)) +@dataclass(frozen=True) +class ReadCount: + """Regional record count and its acquisition receipt; no read BAM is stored.""" + + records: int + receipt: dict + + @dataclass(frozen=True) class AlignmentInfo: """Original header and acquisition evidence; assembly can be unresolved.""" @@ -242,7 +251,7 @@ def _add_mates(regional, mates, inside, output): def _run_bounded(command, output, max_records, timeout): - """Stop indexed acquisition on overflow; never publish a partial BAM.""" + """Stop on overflow; with output=None count records without storing a BAM.""" import pysam command = list(command) command[command.index("-o") + 1] = "-" @@ -261,10 +270,15 @@ def expire(): # AlignmentFile can raise an unraisable seek error when a timeout # interrupts its constructor on a pipe. Read BAM framing forwards # only and copy stored blocks, preserving all original record bytes. - with gzip.GzipFile(fileobj=process.stdout, mode="rb") as bam, pysam.BGZFile(output, "wb") as out: + with gzip.GzipFile(fileobj=process.stdout, mode="rb") as bam, \ + (pysam.BGZFile(output, "wb") if output is not None else nullcontext()) as out: + def write(value): + if out is not None: + out.write(value) + def copy_integer(): value = _read(bam, 4) - out.write(value) + write(value) number, = struct.unpack(" max_records: + if max_records is not None and count > max_records: raise RecordLimitError("Acquisition exceeds record limit") - out.write(size) - out.write(block) + write(size) + write(block) returncode = process.wait() if returncode: stderr.seek(0) @@ -318,7 +332,7 @@ def copy_integer(): if expired.is_set(): stderr.seek(0) raise subprocess.TimeoutExpired(command, timeout, stderr=stderr.read()) - return command + return (command, count) if output is None else command def _samtools_version(): @@ -380,6 +394,40 @@ def _cached_subset(directory, request): return ReadSubset(directory / "reads.bam", directory / "reads.bam.bai", receipt) +def _cached_count(directory, request): + receipt = _verified_receipt(directory, request) + if receipt is None: + return None + if "count.json" not in receipt["files"]: + raise IntegrityError("Cached regional count is missing from its receipt") + try: + count = json.loads((directory / "count.json").read_text())["records"] + except (ValueError, KeyError, TypeError) as error: + raise IntegrityError("Invalid cached regional count") from error + if (type(count) is not int or count < 0 + or type(receipt.get("records")) is not int or count != receipt["records"] + or (request.get("max_records") is not None and count > request["max_records"])): + raise IntegrityError("Cached regional count differs from its receipt or limit") + return ReadCount(count, receipt) + + +def _count_cached_subset(subset, request): + """Verify legacy count metadata against the checksummed BAM's records.""" + import pysam + receipt = subset.receipt + if not {"reads.bam", "reads.bam.bai"} <= receipt["files"].keys(): + raise IntegrityError("Cached regional BAM or index is missing from its receipt") + count = receipt.get("records") + if (type(count) is not int or count < 0 + or (request.get("max_records") is not None and count > request["max_records"])): + raise IntegrityError("Cached regional BAM count differs from its receipt or limit") + with pysam.AlignmentFile(str(subset.path), "rb") as bam: + actual = sum(1 for _ in bam) + if actual != count: + raise IntegrityError("Cached regional BAM count differs from its receipt or limit") + return ReadCount(actual, receipt) + + def _alignment_source(source): file = source if isinstance(source, File) else None location = file.url if file else str(Path(source).resolve()) if not urlsplit(str(source)).scheme else str(source) @@ -466,6 +514,29 @@ def extract_reads(source, regions, *, cache=None, index=None, filters=None, refe max_records stops acquisition on overflow, discards the partial output and raises RecordLimitError; asking again, even offline, raises it without reading. """ + return _read_regions(source, regions, cache=cache, index=index, filters=filters, reference=reference, + fetch_pairs=fetch_pairs, unplaced_mates=unplaced_mates, snapshot_id=snapshot_id, + timeout=timeout, recovery=recovery, max_records=max_records) + + +def count_reads(source, regions, *, cache=None, index=None, filters=None, reference=None, + snapshot_id=None, timeout=600, max_records=None): + """Count indexed regional records without retaining a BAM or fetching mates. + + Return a ReadCount with an acquisition receipt. Overlapping regions form a + union; original duplicate records still count separately. The same reference, + source-identity, filter, timeout and overflow checks as extract_reads apply. + Successful counts and over-limit results are reusable offline. A matching + verified extraction already in the cache can supply the count without I/O + against the original alignment. + """ + return _read_regions(source, regions, cache=cache, index=index, filters=filters, reference=reference, + snapshot_id=snapshot_id, timeout=timeout, max_records=max_records, count_only=True) + + +def _read_regions(source, regions, *, cache=None, index=None, filters=None, reference=None, + fetch_pairs=False, unplaced_mates=True, snapshot_id=None, timeout=600, recovery=None, + max_records=None, count_only=False): if max_records is not None and (type(max_records) is not int or max_records < 1): raise ValueError("max_records must be a positive integer") if recovery is not None: @@ -505,7 +576,7 @@ def extract_reads(source, regions, *, cache=None, index=None, filters=None, refe raise CoordinateError("Reference FASTA must already have a .fai index") local_identities = [file_identity(p) for p in ([] if remote else [location]) + ([] if remote_index else [index])] - request = dict(schema_version=1, operation="extract_reads", + request = dict(schema_version=1, operation="count_reads" if count_only else "extract_reads", source=location if remote else _recorded(location, cache), source_sha256=None if remote else cache.file_digest(location), source_size=file.size if file else None, @@ -523,16 +594,29 @@ def extract_reads(source, regions, *, cache=None, index=None, filters=None, refe if max_records is not None: request["max_records"] = max_records request = json.loads(json.dumps(request)) + cached_result = _cached_count if count_only else _cached_subset directory = cache.workspace / "derived" / stable_id(request) # Reads past max_records aren't kept, but that they were too many is, so asking # again (even offline) gets the same answer without reading them again. over_limit = directory.with_name(directory.name + ".over-limit.json") with file_lock(cache.workspace / "locks" / (directory.name + ".lock")): - cached = _cached_subset(directory, request) + cached = cached_result(directory, request) if cached is not None: return cached if over_limit.exists() and read_read_receipt(over_limit, cache.workspace).get("request") == request: raise RecordLimitError("Acquisition exceeds record limit") + if count_only: + # Reuse completed depth probes made by older builders. Do not delete + # or mutate their BAMs: they may serve other callers too. + old_request = dict(request, operation="extract_reads") + old_directory = directory.with_name(stable_id(old_request)) + with file_lock(cache.workspace / "locks" / (old_directory.name + ".lock")): + old = _cached_subset(old_directory, old_request) + if old is not None: + return _count_cached_subset(old, old_request) + old_limit = old_directory.with_name(old_directory.name + ".over-limit.json") + if old_limit.exists() and read_read_receipt(old_limit, cache.workspace).get("request") == old_request: + raise RecordLimitError("Acquisition exceeds record limit") if remote and cache.offline: raise OfflineError("Regional reads are not cached") require_samtools(fetch_pairs=fetch_pairs, unplaced_mates=unplaced_mates, filters=filters) @@ -576,7 +660,7 @@ def left(): def view(regions_bed, destination, *, name_list=None, pairs=False): """samtools view over these regions, with every filter; checks its output.""" - command = ["samtools", "view", "--no-PG", "-b", "-M", "-X", "-L", str(regions_bed), + command = ["samtools", "view", "--no-PG", "-u" if count_only else "-b", "-M", "-X", "-L", str(regions_bed), "-q", str(filters.min_mapq), "-F", str(filters.exclude_flags), "-f", str(filters.require_flags), "-o", str(destination)] if reference: @@ -588,6 +672,8 @@ def view(regions_bed, destination, *, name_list=None, pairs=False): if name_list: command += ["-N", str(name_list)] command += [location, str(local_index)] + if count_only: + return _run_bounded(command, None, max_records, left()) if max_records is None: _run(command, left()) else: @@ -598,6 +684,9 @@ def view(regions_bed, destination, *, name_list=None, pairs=False): def acquire(): """Read the records. Returns the receipt's commands, and the number of records if it's known.""" + if count_only: + command, count = view(bed, output, name_list=names) + return dict(command=command), count if not fetch_pairs or unplaced_mates: return dict(command=view(bed, output, name_list=names, pairs=fetch_pairs)), None # samtools --fetch-pairs reads the regions a second time, with the positions of @@ -653,7 +742,10 @@ def acquire(): if count is None: with pysam.AlignmentFile(output) as bam: count = sum(1 for _ in bam) - pysam.index(str(output)) + if count_only: + write_json(work / "count.json", dict(records=count)) + else: + pysam.index(str(output)) after = _remote_identity(location, min(timeout, 60)) if remote else None if before != after: raise IntegrityError("Remote alignment changed during extraction") @@ -661,13 +753,15 @@ def acquire(): + ([] if remote_index else [index])] != local_identities: raise IntegrityError("Local alignment or index changed during extraction") (work / "header.sam").write_text(header_text) - if not fetch_pairs: + if count_only: + scope = "regional_record_count" + elif not fetch_pairs: scope = "regional_records" elif unplaced_mates: scope = "regional_records_and_paired_mates" else: scope = "regional_records_and_placed_mates" - files = {name: digest(work / name) for name in ("reads.bam", "reads.bam.bai", "header.sam", "regions.bed", + files = {name: digest(work / name) for name in ("count.json", "reads.bam", "reads.bam.bai", "header.sam", "regions.bed", "mates.bed", "templates.txt") if (work / name).exists()} receipt = dict(request=request, files=files, records=count, inventory_modification=inventory_modification, @@ -686,4 +780,4 @@ def acquire(): # Only a complete directory becomes visible. No receipt means no cache hit. share(work) os.replace(work, directory) - return _cached_subset(directory, request) + return cached_result(directory, request) diff --git a/osteosarc/shared.py b/osteosarc/shared.py index 67b6cbe..970224e 100644 --- a/osteosarc/shared.py +++ b/osteosarc/shared.py @@ -788,7 +788,7 @@ def select(plan): for contig, start, end in elsewhere: region = Region(contig, start, end, assembly, lengths[contig] if contig in MITOCHONDRIA else None) try: # a region in a pileup (a repeat that draws millions of reads) costs more than it's worth - dataset.extract_reads(plan["file"], [region], max_records=split_depth) + dataset.count_reads(plan["file"], [region], max_records=split_depth) added.append(region) except RecordLimitError: crowded += 1 diff --git a/osteosarc/views.py b/osteosarc/views.py index f97ca90..0ad43c4 100644 --- a/osteosarc/views.py +++ b/osteosarc/views.py @@ -6,11 +6,13 @@ from __future__ import annotations +import shlex import shutil import textwrap from collections import Counter, defaultdict from pathlib import PurePosixPath +from .catalog import fastq_prefix, matches_fastq from .curation import ASSAY_NAMES, ASSAYS, normalize_provider from .errors import OsteosarcError from .urls import BUCKET, S3_BUCKET @@ -167,8 +169,8 @@ def samples_view(samples, *, width=None, footer=None): f"{len(samples)} samples: " + ", ".join(f"{n} {t}" for t, n in tissues)) rows = [dict(sample=s.id, timepoint=s.timepoint or "", date=s.date, where=s.site, sequencing=sequencing_text(s.sequencing) or "none recorded", BAMs=len(s.bams), - **{"FASTQ folders": len(s.fastq_folders)}) for s in samples] - shown = table(rows, ("sample", "timepoint", "date", "where", "sequencing", "BAMs", "FASTQ folders"), + **{"FASTQ locations": len(s.fastq_folders)}) for s in samples] + shown = table(rows, ("sample", "timepoint", "date", "where", "sequencing", "BAMs", "FASTQ locations"), width=width, wrap=True) footer = footer if footer is not None else hints([ ('data.samples["T1_tumor"]', "one sample's files, with how to get them"), @@ -183,30 +185,25 @@ def samples_view(samples, *, width=None, footer=None): def _fastq_folders(sample, files): - """One row per FASTQ folder: its assay and platform (the names file filters use), - library, provider, and files and bytes in the bucket.""" + """One row per published FASTQ location, counting only matching raw reads.""" labels = {} for row in sample.details.get("fastqs", ()): labels.setdefault(row["folder"], (row.get("assay") or "", row.get("provider") or "")) - counts, sizes = Counter(), Counter() - if files is not None: - folders = set(sample.fastq_folders) - for file in files: - parts = file.key.split("/") - for depth in range(len(parts) - 1, 0, -1): - folder = "/".join(parts[:depth]) - if folder in folders: - counts[folder] += 1 - sizes[folder] += file.size or 0 - break + reads = [f for f in files or () if f.kind == "reads"] rows = [] for folder in sample.fastq_folders: label, provider = labels.get(folder, ("", "")) assay, platform = ASSAYS.get(label, (label, None)) + matched = [f for f in reads if matches_fastq(f.key, folder)] + prefix = fastq_prefix(folder) + directory = not matched or all(f.key.startswith(prefix + "/") for f in matched) rows.append(dict(assay=assay, platform=platform or "", library=LIBRARIES.get(label, ""), provider=normalize_provider(provider) if provider else "", - files=counts[folder] if files is not None else "", - size=size_text(sizes[folder]), folder=folder + "/")) + files=len(matched) if files is not None else "", + size=size_text(sum(f.size or 0 for f in matched)), + folder=prefix + ("/" if directory else ""), + published_location=folder, directory=directory, + keys=tuple(f.key for f in matched))) order = {name: i for i, name in enumerate(ASSAY_NAMES)} kinds = {name: i for i, name in enumerate(dict.fromkeys(LIBRARIES.values()))} return sorted(rows, key=lambda r: (order.get(r["assay"], len(order)), r["assay"], @@ -250,7 +247,7 @@ def sample_files_view(sample, data, *, width=None, local=None): folders = _fastq_folders(sample, data.files.select(sample=sample.id)) lines.append("") if folders: - lines.append(f"FASTQ folders, raw reads ({len(folders)}):") + lines.append(f"FASTQ locations, raw reads ({len(folders)}; folders or filename prefixes):") lines.append(table(folders, ("assay", "platform", "library", "provider", "files", "size", "folder"), width=width, fixed=("folder", "library"), drop_empty=True)) else: @@ -303,12 +300,21 @@ def get_data_hints(data, bams, folders, *, python=False): f"the whole BAM ({bam['size'] or 'size unknown'})" + (" and its index" if bam["indexed"] else "") + ", into this folder")) if folders: - folder = folders[0]["folder"] + location = folders[0] + folder = location["folder"] lines.append((f'data.files.select(prefix="{folder}")' if python else f"osteosarc files --prefix {folder}", - "the files in a FASTQ folder")) + "files under this FASTQ location")) if data._download_header.get("download_base", BUCKET) == BUCKET: # not for a mirror - lines.append((f"aws s3 cp --recursive --no-sign-request {S3_BUCKET}{folder} " - f"{PurePosixPath(folder).name}/", "a whole folder, with the AWS CLI (in a shell)")) + if location.get("directory", folder.endswith("/")): + lines.append((f"aws s3 cp --recursive --no-sign-request {shlex.quote(S3_BUCKET + folder)} " + f"{shlex.quote(PurePosixPath(folder).name + '/')}", + "a whole folder, with the AWS CLI (in a shell)")) + elif location["keys"]: + parent = str(PurePosixPath(folder).parent) + "/" + includes = " ".join("--include " + shlex.quote(key[len(parent):]) for key in location["keys"]) + lines.append((f"aws s3 cp --recursive --no-sign-request {shlex.quote(S3_BUCKET + parent)} " + f"./ --exclude '*' {includes}", + "only this location's FASTQs, with the AWS CLI (in a shell)")) if not lines: return " (no files)" return "\n".join(f" {command}\n {what}" for command, what in lines) diff --git a/tests/data/comprehensive/README.md b/tests/data/comprehensive/README.md index 29d4d81..6175e4a 100644 --- a/tests/data/comprehensive/README.md +++ b/tests/data/comprehensive/README.md @@ -90,6 +90,11 @@ The read audit also verifies preservation of every selected historical NR2F2 record and identifies the current library members containing them. Its seven historical ALT templates use the original, unfiltered classifier; they are not claimed to pass the independent MAPQ/NH/base-quality policy below. +An offline recount on 2026-10-05 of the original unsampled GRCh37 extract found +81 overlapping records: 79 failed the MAPQ-255/NH policy and two had MAPQ below +20. No templates passed those filters, so that recount supplies no high-quality +reference or alternate evidence. It does not invalidate the seven historical +unfiltered ALT templates or turn NR2F2 into a negative control. The read audit independently recounts these loci from **full regional extracts before fixture sampling**. It collapses RG/QNAME templates, excludes duplicate, @@ -172,6 +177,9 @@ The comprehensive selection permits up to 2,000 supplementary regions per source Supplementary-alignment retrieval permits up to 1,000,000 records per queried region; MAPQ-zero/invalid SA entries, absent contigs and over-limit regions are excluded and reported by the acquisition log. +Depth probes stream counts without storing a BAM. Their receipts, limits and +source-identity checks remain cached, and verified extracts from earlier attempts +can supply the same counts. This reduces disk use without changing selection. Read fixtures are deliberately balanced and capped, and are **unsuitable for abundance or VAF estimation**. Use the published support matrix or pre-sampling diff --git a/tests/data/curation-review-2026-10-05.json b/tests/data/curation-review-2026-10-05.json new file mode 100644 index 0000000..d102e57 --- /dev/null +++ b/tests/data/curation-review-2026-10-05.json @@ -0,0 +1,791 @@ +{ + "snapshot": { + "name": "2026-10-05", + "id": "ac82984eff4a59fcc411bb05538fb01cb16bcdd387152aa4e78d9bdc31047946" + }, + "sources": { + "bam_metadata": [ + { + "display_name": "T0 Tempus Tumor Panel 2022-12 (TL-24-5GQLV9WSXQ xT 648-gene vendor)", + "tissue": "Tumor", + "assay": "Panel", + "timepoint": "T0", + "sample_date": "2022-12", + "provider": "Tempus", + "s3_path": "vendor/tempus/TL-24-5GQLV9WSXQ/DNA/TL-24-5GQLV9WSXQ_T.sorted.bam" + }, + { + "display_name": "T0 Tempus Tumor WES 2022-12 (TL-24-ALMY2X4KMV vendor)", + "tissue": "Tumor", + "assay": "WES", + "timepoint": "T0", + "sample_date": "2022-12", + "provider": "Tempus", + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV_T.sorted.bam" + }, + { + "display_name": "Tempus Blood WES 2024-09-21 (TL-24-ALMY2X4KMV vendor)", + "tissue": "Blood", + "assay": "WES", + "timepoint": "", + "sample_date": "2024-09-21", + "provider": "Tempus", + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV_N.sorted.bam" + }, + { + "display_name": "T0 Tempus Tumor RNA 2022-12 (TL-24-ALMY2X4KMV vendor)", + "tissue": "Tumor", + "assay": "RNA", + "timepoint": "T0", + "sample_date": "2022-12", + "provider": "Tempus", + "s3_path": "vendor/tempus/TL-24-ALMY2X4KMV/RNA/TL-24-ALMY2X4KMV_T_sorted.bam" + }, + { + "display_name": "T0 Tempus Tumor RNA 2022-12 (TL-24-KCVBE1UI1P vendor)", + "tissue": "Tumor", + "assay": "RNA", + "timepoint": "T0", + "sample_date": "2022-12", + "provider": "Tempus", + "s3_path": "vendor/tempus/TL-24-KCVBE1UI1P/RNA/TL-24-KCVBE1UI1P_T_sorted.bam" + } + ], + "vafs": [ + { + "variant_id": "PDZRN4-chr12-41572809", + "gene": "PDZRN4", + "chrom": "chr12", + "pos": "41572809", + "chr_pos": "chr12:41572809", + "ref": "A", + "alt": "T", + "change": "A>T", + "variant_type": "snv", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "242", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "242", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "PDZRN4-chr12-41572809", + "gene": "PDZRN4", + "chrom": "chr12", + "pos": "41572809", + "chr_pos": "chr12:41572809", + "ref": "A", + "alt": "T", + "change": "A>T", + "variant_type": "snv", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "242", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "242", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "TRMO-chr9-97910412", + "gene": "TRMO", + "chrom": "chr9", + "pos": "97910412", + "chr_pos": "chr9:97910412", + "ref": "C", + "alt": "T", + "change": "C>T", + "variant_type": "snv", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "393", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "393", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "TRMO-chr9-97910412", + "gene": "TRMO", + "chrom": "chr9", + "pos": "97910412", + "chr_pos": "chr9:97910412", + "ref": "C", + "alt": "T", + "change": "C>T", + "variant_type": "snv", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "393", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "393", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "DCHS2-chr4-154323273", + "gene": "DCHS2", + "chrom": "chr4", + "pos": "154323273", + "chr_pos": "chr4:154323273", + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "change": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC>TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "variant_type": "deletion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "DCHS2-chr4-154323273", + "gene": "DCHS2", + "chrom": "chr4", + "pos": "154323273", + "chr_pos": "chr4:154323273", + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "change": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC>TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "variant_type": "deletion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "GAPVD1-chr9-125301980", + "gene": "GAPVD1", + "chrom": "chr9", + "pos": "125301980", + "chr_pos": "chr9:125301980", + "ref": "TAGTGC", + "alt": "ATTGG", + "change": "TAGTGC>ATTGG", + "variant_type": "deletion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "GAPVD1-chr9-125301980", + "gene": "GAPVD1", + "chrom": "chr9", + "pos": "125301980", + "chr_pos": "chr9:125301980", + "ref": "TAGTGC", + "alt": "ATTGG", + "change": "TAGTGC>ATTGG", + "variant_type": "deletion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "MAP2-chr2-209694768", + "gene": "MAP2", + "chrom": "chr2", + "pos": "209694768", + "chr_pos": "chr2:209694768", + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG", + "change": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT>CAGGG", + "variant_type": "deletion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "390", + "alt_reads": "0", + "other_reads": "1", + "total_reads": "391", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "MAP2-chr2-209694768", + "gene": "MAP2", + "chrom": "chr2", + "pos": "209694768", + "chr_pos": "chr2:209694768", + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG", + "change": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT>CAGGG", + "variant_type": "deletion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "390", + "alt_reads": "0", + "other_reads": "1", + "total_reads": "391", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "USH2A-chr1-215650752", + "gene": "USH2A", + "chrom": "chr1", + "pos": "215650752", + "chr_pos": "chr1:215650752", + "ref": "C", + "alt": "A", + "change": "C>A", + "variant_type": "snv", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "90", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "90", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "USH2A-chr1-215650752", + "gene": "USH2A", + "chrom": "chr1", + "pos": "215650752", + "chr_pos": "chr1:215650752", + "ref": "C", + "alt": "A", + "change": "C>A", + "variant_type": "snv", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "90", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "90", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "COL3A1-Splice", + "gene": "COL3A1", + "chrom": "chr2", + "pos": "189010889", + "chr_pos": "chr2:189010889", + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C", + "change": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA>C", + "variant_type": "deletion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "COL3A1-Splice", + "gene": "COL3A1", + "chrom": "chr2", + "pos": "189010889", + "chr_pos": "chr2:189010889", + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C", + "change": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA>C", + "variant_type": "deletion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "FAM157A-p_W70_Q71ins_14", + "gene": "FAM157A", + "chrom": "chr3", + "pos": "198153259", + "chr_pos": "chr3:198153259", + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "change": "G>GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "variant_type": "insertion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "FAM157A-p_W70_Q71ins_14", + "gene": "FAM157A", + "chrom": "chr3", + "pos": "198153259", + "chr_pos": "chr3:198153259", + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "change": "G>GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "variant_type": "insertion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "", + "alt_reads": "", + "other_reads": "", + "total_reads": "", + "vaf": "", + "status": "unsupported" + }, + { + "variant_id": "OTUD4-p_A153del", + "gene": "OTUD4", + "chrom": "chr4", + "pos": "145155970", + "chr_pos": "chr4:145155970", + "ref": "TCAG", + "alt": "T", + "change": "TCAG>T", + "variant_type": "deletion", + "bam_file": "normal_bg22.recal.bam", + "ref_reads": "56", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "56", + "vaf": "0.0000", + "status": "ok" + }, + { + "variant_id": "OTUD4-p_A153del", + "gene": "OTUD4", + "chrom": "chr4", + "pos": "145155970", + "chr_pos": "chr4:145155970", + "ref": "TCAG", + "alt": "T", + "change": "TCAG>T", + "variant_type": "deletion", + "bam_file": "normal_bg24.recal.bam", + "ref_reads": "56", + "alt_reads": "0", + "other_reads": "0", + "total_reads": "56", + "vaf": "0.0000", + "status": "ok" + } + ], + "source_variants": [ + { + "id": "TRMO-chr9-97910412", + "gene": "TRMO", + "chr": "chr9", + "pos": 97910412, + "ref": "C", + "alt": "T", + "genomic_location": "chr9:97910412", + "genomic_change_on_cdna": "C>T", + "genomic_ref_context": "CTCTCAGGGTGTGAT", + "refseq_id": "NM_001371657", + "variant_type": "snv", + "protein_change": "p.Gly205Glu", + "note": null + }, + { + "id": "DCHS2-chr4-154323273", + "gene": "DCHS2", + "chr": "chr4", + "pos": 154323273, + "ref": "GTTTTTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTTCAGAGGCATAGGTCTAGCTGCC", + "alt": "TTTGCACGACTGCTTCCCAAATGCTGTTTTTCCCTACCGAGGCATATGTCTAGCTGCCAA", + "genomic_location": "chr4:154323273", + "genomic_change_on_cdna": "c.4019-5_4075delins(60)", + "genomic_ref_context": "CAAAAAACAAACAAA", + "refseq_id": "NM_001142552", + "variant_type": "complex", + "protein_change": "p.Ala1340_Gln1359delins(20)", + "note": "Tempus xE 2022 pindel complex call (GRCh37 4:155244425, lifted ungapped). The vendor allele also contains two homozygous germline variants (rs140019361 GTTT contraction; rs12500437 p.P1342H). The only candidate somatic changes, chr4:154323312 T>A (c.4036A>T p.K1346*) and 154323314 A>C (c.4034T>G p.L1345R), are absent from all other tumor sequencing (0/576 reads BostonGene T0 WES, 0/95 Personalis T0 WGS, 0 at T1/T2) and seen in 1 read of Tempus's own tumor BAM. Probably a Tempus artefact; read counts not available." + }, + { + "id": "GAPVD1-chr9-125301980", + "gene": "GAPVD1", + "chr": "chr9", + "pos": 125301980, + "ref": "TAGTGC", + "alt": "ATTGG", + "genomic_location": "chr9:125301980", + "genomic_change_on_cdna": "c.186-3_188delinsATTGG", + "genomic_ref_context": "TTTTTTTTAGTGCTG", + "refseq_id": "NM_015635", + "variant_type": "complex", + "protein_change": "p.Ser62fs", + "note": "Tempus xE 2022 pindel call (GRCh37 9:128064259 TAGTGC>ATTGG, 10.8%) at the exon 3 splice acceptor (intron 2/exon 3 boundary) after a poly-T tract. No reads with this allele in any other tumor sequencing (0/108 BostonGene T0 WES, 0/37 Personalis T0 WGS, 0 at T1/T2); 1 read in Tempus's own tumor BAM. Partial G>T reads appear in tumor and normal alike. Probably a Tempus artefact; read counts not available." + }, + { + "id": "MAP2-chr2-209694768", + "gene": "MAP2", + "chr": "chr2", + "pos": 209694768, + "ref": "CCTGGGCTACTGTGTGTTCAATAAGTACACAGT", + "alt": "CAGGG", + "genomic_location": "chr2:209694768", + "genomic_change_on_cdna": "c.2599_2630delinsAGGG", + "genomic_ref_context": "TCGAAGACCTGGGCT", + "refseq_id": "NM_002374", + "variant_type": "delins", + "protein_change": "p.Leu867fs", + "note": "One complex event, NM_002374.4:c.2599_2630delinsAGGG p.(Leu867ArgfsTer39); minimal VCF form chr2:209694769 CTGGGCTACTGTGTGTTCAATAAGTACACAGT>AGGG, written here anchored at 209694768. Tempus (freebayes/pindel) and DRAGEN call it as one delins; CeGaT (c.2599C>A + c.2600T>G + c.2603_2630del) and oncoanalyser (CT>AG at 209694769 + 28-bp deletion at 209694772) split it; sarek Mutect2 emits three phased records (22-bp del + GTACACA>G + T>G, FILTER clustered_events;haplotype), the first of which was previously stored here. The mRNA and JLF peptides derive from this allele." + }, + { + "id": "FAM157A-p_W70_Q71ins_14", + "gene": "FAM157A", + "chr": "chr3", + "pos": 198153259, + "ref": "G", + "alt": "GGCGGCGGCGGCGGCAGCAGCAGCAGCAGCAGCAGCAGCAGCA", + "genomic_location": "chr3:198153259", + "genomic_change_on_cdna": "c.211_212ins(42)", + "genomic_ref_context": "AAGAACTGGCAGCAG", + "refseq_id": "NM_001145248", + "variant_type": "insertion", + "protein_change": "p.Trp70_Gln71ins(14)", + "note": "Tempus pindel GRCh37 3:197880130 G>G+42 (5/39 tumor reads, 0/30 normal), lifted to GRCh38 chr3:198153259. The site is the germline-polymorphic CAG repeat rs56683636, where the patient's normal DNA is heterozygous for 6-bp and 9-bp deletions, so this is a low-confidence Tempus-only call; the allele is not countable by the site pileup. Protein caveat: NCBI has suppressed NM_001145248.1 (support for the transcript but not for the protein) and now annotates FAM157A as non-coding (NR_146164.1, whose exon 1 contains this insertion), so p.W70_Q71ins(14) derives from a retired protein model and is not supported by current annotation." + }, + { + "id": "USH2A-chr1-215650752", + "gene": "USH2A", + "chr": "chr1", + "pos": 215650752, + "ref": "C", + "alt": "A", + "genomic_location": "chr1:215650752", + "genomic_change_on_cdna": "c.14183G>T", + "genomic_ref_context": "ACATGGTGCAGAACC", + "refseq_id": "NM_206933", + "variant_type": "snv", + "protein_change": "p.Cys4728Phe", + "note": null + }, + { + "id": "COL3A1-Splice", + "gene": "COL3A1", + "chr": "chr2", + "pos": 189010889, + "ref": "CGGTAGGAAACATTTTTCTCAATATAGGTCATAAAGCAGTCAGCATTTTAGTTTAATCATGCAAATTATTTTGAATAGAATAAATAAAATTAATAGGATGAAATAAAGATAGCATTTGGTATGAATTACATACATTGCATCTACTGATTCATTGCAGGGAATGTTAAATGCAACAAAATGGATCTTAGCCTCCAGATGAAAACCAGTTGAAATAAAAGCATTTAAAATTTAAATCCATAATGCAAACTTTATCAGATAATTGGGATAGTTACTATATGTTTTGAATAATACCTGTCACATTAACTCAGTTTGAAATATCTGTTTTTTAAAATTTTAAAGGTTAACTTCAAATCTCTCATTTGTTATTGTTATTTGTGAACAAAGGGAAAACTCACTTCTAATATTAGCGAATTTCATTGTGAGAGACCTATCCTCTTTTTAATAAACCATTTATAAACCTGTTAGCAATGGCCGGAACCAGGCCTCCTGAGGATGCACTGGTCTATAGCAATTGCCCTGCTGCATTTACTAAGAATCCTTACAAGGCATTTGTTTAAGAATATTGTTTATCAACTAAGAAGATTACAGCTTTGAAGTAGAGCAGGTCTCATATACATGAATAATAACATGGCACGATGAATGCTTCTTTAGAGTAAAAAGGTTTTCTTTAACTTGTTAAGTCAGAGTTGTCTAAGTAATTGTAATGTCATGATCATGTACATTTTGTCCTTTTTTACA", + "alt": "C", + "genomic_location": "chr2:189010889", + "genomic_change_on_cdna": "c.4254+1_4255-1del", + "genomic_ref_context": "GGTTGCACGGTAGGA", + "refseq_id": "NM_000090", + "variant_type": "deletion", + "protein_change": "Splice", + "note": "737-bp deletion of intron 50 (NM_000090.4 c.4254+1_4255-1del). Source: Tempus xE pindel VCF (GRCh37 2:189875615; tumor 31/221, normal 0/212). Natera reports 'COL3A1 Breakpoint: Deletion'; the Natera tumor BAM carries the same exon50-exon51 junction. Split-read support in the 2022 tumor WGS/WES, absent in normals and later tumors. Not countable by the site pileup." + }, + { + "id": "OTUD4-p_A153del", + "gene": "OTUD4", + "chr": "chr4", + "pos": 145155970, + "ref": "TCAG", + "alt": "T", + "genomic_location": "chr4:145155970", + "genomic_change_on_cdna": "c.458_460del", + "genomic_ref_context": "GCTGCTGATGTGAAT", + "refseq_id": "NM_001102653", + "variant_type": "deletion", + "protein_change": "p.Ala153del", + "note": "Natera p.A153del (NM_001102653.1) = one Ala codon lost from a pure (CTG)6 repeat (hg19 4:146077122; public CeGaT normal-DNA call 7/323, flagged FP/HET). Natera tumor BAM ~6% vs 0 in Natera germline, but the deletion also appears in the Tempus matched normal (11/208) and CeGaT normal, so somatic status is uncertain (possible replication slippage)." + } + ], + "variant_index": [ + { + "id": "TRMO-chr9-97910412", + "gene": "TRMO", + "location": "chr9:97910412", + "protein_label": "p.Gly205Glu" + }, + { + "id": "DCHS2-chr4-154323273", + "gene": "DCHS2", + "location": "chr4:154323273", + "protein_label": "p.Ala1340_Gln1359delins(20)" + }, + { + "id": "GAPVD1-chr9-125301980", + "gene": "GAPVD1", + "location": "chr9:125301980", + "protein_label": "p.Ser62fs" + }, + { + "id": "FAM157A-p_W70_Q71ins_14", + "gene": "FAM157A", + "location": "chr3:198153259", + "protein_label": "p.Trp70_Gln71ins(14)" + }, + { + "id": "USH2A-chr1-215650752", + "gene": "USH2A", + "location": "chr1:215650752", + "protein_label": "p.Cys4728Phe" + }, + { + "id": "COL3A1-Splice", + "gene": "COL3A1", + "location": "chr2:189010889", + "protein_label": "Splice" + }, + { + "id": "OTUD4-p_A153del", + "gene": "OTUD4", + "location": "chr4:145155970", + "protein_label": "p.Ala153del" + }, + { + "id": "MAP2-chr2-209694768", + "gene": "MAP2", + "location": "chr2:209694768", + "protein_label": "p.Leu867fs" + } + ], + "specimens": [ + { + "sample_id": "T1_tumor", + "collection_date": "2024-06-06", + "collection_site": "UCLA", + "tissue_source": "Recurrent tumor biopsy (CT-guided, T4-T5)" + }, + { + "sample_id": "T3_tumor", + "collection_date": "2025-04-17", + "collection_site": "MSKCC", + "tissue_source": "Tumor resection (T4-T5)" + }, + { + "sample_id": "T3_tumor_CD45neg", + "collection_date": "2025-04-17", + "collection_site": "MSKCC", + "tissue_source": "CD45-negative enriched fraction of T3 tumor" + } + ], + "events": [ + { + "date": "2026-06-15", + "title": "ReyaGel", + "end_date": "2026-07-14" + } + ], + "events_sheet": [ + { + "Title": "ReyaGel", + "Start date": "06/15/2026", + "End date": "07/14/2026" + } + ], + "bams": [] + }, + "receipts": { + "bam_metadata": { + "etag": "\"b288e082de0413ca220ef852e48c382d\"", + "filename": "bam-metadata-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:34.206463+00:00", + "sha256": "cb620face7a055283818eea668a18695d12dc280e0ca7666cb637d37938da2ef", + "size": 30927, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/bam-metadata-consolidated.tsv" + }, + "bams": { + "etag": "\"dlw6xutusq7h11yk\"", + "filename": "bams.json", + "last_modified": "Sun, 04 Oct 2026 16:22:55 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:28.732350+00:00", + "sha256": "012d031eb824e43aa2dff8991334bc6078677f132f2c50733b31a1b4a321fe6b", + "size": 49196, + "url": "https://osteosarc.com/bams/bams.json" + }, + "bucket": { + "etag": "\"dlw6xuxtow2vqnr7f\"", + "filename": "bucket_listing.json", + "last_modified": "Sun, 04 Oct 2026 16:22:55 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:29.957335+00:00", + "sha256": "3567118cd3e3ee6dd5533916da86079e40028427db92ba150c8ee7ee1d0d8d12", + "size": 44778363, + "url": "https://osteosarc.com/bucket_listing.json" + }, + "cytometry": { + "etag": "\"dlw6xvikwks82931\"", + "filename": "cytometry.tsv", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:41.024104+00:00", + "sha256": "c1c20370ab67eddfdef60909e6fd7ca1ec47bdf4d17ddfb18f1064e69f90f055", + "size": 105085, + "url": "https://osteosarc.com/data/cytometry.tsv" + }, + "data_page": { + "etag": null, + "filename": "data", + "last_modified": "Sun, 04 Oct 2026 16:23:05 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:34.716654+00:00", + "sha256": "a586a22add933e7ba93ee74f6e951f5b9fc9806436cc68f9be3dc3f3866357f6", + "size": 131895, + "url": "https://osteosarc.com/data/" + }, + "events": { + "etag": "\"dlw6xvikwks81ilk\"", + "filename": "events.json", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:35.193016+00:00", + "sha256": "fcc9fd183649e9880476b36e320d3c48f5397f0252385041dae5a51209a342b3", + "size": 70760, + "url": "https://osteosarc.com/data/events.json" + }, + "events_sheet": { + "etag": "\"4463bc4d9822fa2c42891553605585b6\"", + "filename": "timeline.csv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:35.781676+00:00", + "sha256": "d260caafb1e92eb02d4ead8d19c7ae7267fdcec55bbac9d9490d0701d7b61cfd", + "size": 25182, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/timeline/timeline.csv" + }, + "fastqs": { + "etag": "\"ddcf14af8eeb0138c561edfa83ba653a\"", + "filename": "fastqs-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:38.529109+00:00", + "sha256": "4e7aed516e48f448262370eb23103603f10cae69296c920a2e60e6727c10b762", + "size": 75106, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/fastqs-consolidated.tsv" + }, + "flow": { + "etag": "\"dlw6xvim3g17ahe\"", + "filename": "manifest.json", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:38.935073+00:00", + "sha256": "492a996c95f625a9490ff62e06d56079d741fdd10ce1232c79a7515a5a7a5b04", + "size": 13586, + "url": "https://osteosarc.com/data/flow/manifest.json" + }, + "imaging": { + "etag": "\"995dff34f1ac47269e905f95e08f83d6\"", + "filename": "dicom-studies.json", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:39.580784+00:00", + "sha256": "e9710af61cd4b062238e6de1d1a96543e67ce8c9ee50c89ac180152430eb6cfc", + "size": 732839, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/src/data/dicom-studies.json" + }, + "labs": { + "etag": "\"dlw6xvje2udscoj5\"", + "filename": "lab_results.tsv", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:40.546253+00:00", + "sha256": "b9e503df90af8ea925c21a88cf64f893b7cc14cdd8faa9bdc867eab1f307f5a4", + "size": 591665, + "url": "https://osteosarc.com/data/lab_results.tsv" + }, + "mrd": { + "etag": "\"dlw6xvjiubdn8jx\"", + "filename": "mrd.json", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:36.114377+00:00", + "sha256": "28e3852e05d1178c3ad23c52fcfdfea836ae30a08a519aff1b6e6d790a422926", + "size": 11085, + "url": "https://osteosarc.com/data/mrd.json" + }, + "pathology": { + "etag": "\"6124c904f0138e7b5c0dbf0ee1691424\"", + "filename": "pathology-slides.json", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:40.094434+00:00", + "sha256": "6270b51818a0783e47b7307cea8e042bcff15be3d0d2df4965671d15713758d3", + "size": 17243, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/src/data/pathology-slides.json" + }, + "source_variants": { + "etag": "\"1f225c84178db9e153a59558e0de1fa7\"", + "filename": "variants.json", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:33.471333+00:00", + "sha256": "15346cb02091853841e29cfd6c24d54e6c0303f98b4c83fd2f2f32c5bb734698", + "size": 7695848, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/src/data/variants.json" + }, + "specimens": { + "etag": "\"65a4a32376236a8d4dbee439f79fd8ed\"", + "filename": "samples-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:36.928173+00:00", + "sha256": "245e78a7ff1bd9f9713cd1e356c24cc803c2f3d0cfdce88da6e043791547029d", + "size": 17386, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/samples-consolidated.tsv" + }, + "timepoint_summary": { + "etag": "\"63fcaa2172297d43948c0d8fba778241\"", + "filename": "samples.json", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:37.494323+00:00", + "sha256": "e87ce665a5d8cb8550af2feac5c35d5955917eba1ea11624a85d6ce8bc9c00ce", + "size": 614, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/src/data/samples.json" + }, + "vaccine_overlap": { + "etag": "\"dlw6xvjrrtqupp1\"", + "filename": "vaccine_overlap.json", + "last_modified": "Sun, 04 Oct 2026 16:22:57 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:32.068622+00:00", + "sha256": "ba365f8cbe3889b17eaedb642c0ef35ee9f61b6e24f56f8362530eb596530886", + "size": 33301, + "url": "https://osteosarc.com/data/vaccine_overlap.json" + }, + "vaf_columns": { + "etag": "\"dlw6xwnuzvg83ge\"", + "filename": "variant_vafs_long.columns.tsv", + "last_modified": "Sun, 04 Oct 2026 16:22:59 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:31.648247+00:00", + "sha256": "1a1df2feb5adec8b38ee34a95b3f6a6216fe91ea24ad0650ce7b293794a137c0", + "size": 4478, + "url": "https://osteosarc.com/variants/variant_vafs_long.columns.tsv" + }, + "vafs": { + "etag": "\"dlw6xwnykh7417sxo\"", + "filename": "variant_vafs_long.tsv", + "last_modified": "Sun, 04 Oct 2026 16:22:59 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:31.050983+00:00", + "sha256": "dd743e698c6239bdc9d11558e18b3d171bf6618bcfca09e1a3ac51818fad599e", + "size": 2043708, + "url": "https://osteosarc.com/variants/variant_vafs_long.tsv" + }, + "variant_index": { + "etag": null, + "filename": "variants", + "last_modified": "Sun, 04 Oct 2026 16:23:07 GMT", + "md5": null, + "retrieved_at": "2026-10-05T14:27:30.448507+00:00", + "sha256": "34e9adc1687b02660a6d1cdb55a5e36b2f9725c2ef5b4ad27b3c316cf362c392", + "size": 602733, + "url": "https://osteosarc.com/variants/" + } + }, + "selection": "Affected source records and the first two published VAF rows per variant; source absence checks cover the complete source lists. The full live snapshot was separately checked with corrections --strict." +} diff --git a/tests/data/otud4-review/NC_000004.11.fasta b/tests/data/otud4-review/NC_000004.11.fasta new file mode 100644 index 0000000..e492370 --- /dev/null +++ b/tests/data/otud4-review/NC_000004.11.fasta @@ -0,0 +1,17 @@ +>NC_000004.11:146076622-146077622 Homo sapiens chromosome 4, GRCh37.p13 Primary Assembly +GCTGAGCTGTTAAAAAAAAAAAGGTCAGTATAATATAAAGTTGACTTATTTTCAAGTGCTACAAAGCAAA +TTTATGGAAAATGCAGATTTTTAACTCACCTTGTTTAGACTTCAGCCAAATTTCATATTCCACATTTCTA +TAGACTGCAGGATTGAGTGACTTAAGAACCTTTCTAGACAAAGGCAGGCTAGTAGAGTTCCCATTGTTCT +TCAGCTGCTAAACAAAGTCAGGAAGTCCAATCAACACATAAGTGCTTTTAAAACATTTTTGTTTGAGATA +CACGTAAAACTAAAGTCAAATGAAGAAAAGAAACTTCAAAAAATTAGGAAGCCACCAAATCAATTTCTGA +GTAGATGATCCTGACTTGACTGAAATGTACCAGAAAAAAGAAAAAAATTAAGATTTTAATGTACATGCTT +TCTTATATTAAAGAACTACATTTAAAACCACTTTTAAAAAAAATACCTCATTGCCTGACAAAGGTTTAAA +TCCATTCACATCAGCAGCAGCAGCAGCAGTCTTACTCCTACAAAGAAAGATAACCATAATTGGGGCAGAA +AAAGGTATTTGCACGTAATTACCTTCTAACCTCATCTAAACGTCTTCAAAGTCAGAACTATGCTCCAAAA +AGAGCTATATTCCAGTGAGCTCATACAAGTTTTGGTACAAGTATATCTATATATAAAAGGGTATATGTAT +GTAAATGTGTGTATATAAGACAGAAGTCAAAGACAACAGTAATTCTAACTGGCTCAGTAATAAGAAAACA +TAAACTGTGAAATTTGGGTGGGTAGAATAATTGAATTTACAAAAGAATGTAAAAACTTCCTTTTAATATA +CACATGAGGCCCCGTTTAAAAACACAATCCATTTACAACAAAAGTATCCATAGGCAGATTCGACAACCCT +TTTATTAAAAATACTCTACATCAGCCGGCCATGGTGGCTCACGCCTGTAATCCCAGGACTTTGGGAGGCA +CAGGCAGGCAGATCACCTGAG + diff --git a/tests/data/otud4-review/NC_000004.12.fasta b/tests/data/otud4-review/NC_000004.12.fasta new file mode 100644 index 0000000..4285f74 --- /dev/null +++ b/tests/data/otud4-review/NC_000004.12.fasta @@ -0,0 +1,17 @@ +>NC_000004.12:145155470-145156470 Homo sapiens chromosome 4, GRCh38.p14 Primary Assembly +GCTGAGCTGTTAAAAAAAAAAAGGTCAGTATAATATAAAGTTGACTTATTTTCAAGTGCTACAAAGCAAA +TTTATGGAAAATGCAGATTTTTAACTCACCTTGTTTAGACTTCAGCCAAATTTCATATTCCACATTTCTA +TAGACTGCAGGATTGAGTGACTTAAGAACCTTTCTAGACAAAGGCAGGCTAGTAGAGTTCCCATTGTTCT +TCAGCTGCTAAACAAAGTCAGGAAGTCCAATCAACACATAAGTGCTTTTAAAACATTTTTGTTTGAGATA +CACGTAAAACTAAAGTCAAATGAAGAAAAGAAACTTCAAAAAATTAGGAAGCCACCAAATCAATTTCTGA +GTAGATGATCCTGACTTGACTGAAATGTACCAGAAAAAAGAAAAAAATTAAGATTTTAATGTACATGCTT +TCTTATATTAAAGAACTACATTTAAAACCACTTTTAAAAAAAATACCTCATTGCCTGACAAAGGTTTAAA +TCCATTCACATCAGCAGCAGCAGCAGCAGTCTTACTCCTACAAAGAAAGATAACCATAATTGGGGCAGAA +AAAGGTATTTGCACGTAATTACCTTCTAACCTCATCTAAACGTCTTCAAAGTCAGAACTATGCTCCAAAA +AGAGCTATATTCCAGTGAGCTCATACAAGTTTTGGTACAAGTATATCTATATATAAAAGGGTATATGTAT +GTAAATGTGTGTATATAAGACAGAAGTCAAAGACAACAGTAATTCTAACTGGCTCAGTAATAAGAAAACA +TAAACTGTGAAATTTGGGTGGGTAGAATAATTGAATTTACAAAAGAATGTAAAAACTTCCTTTTAATATA +CACATGAGGCCCCGTTTAAAAACACAATCCATTTACAACAAAAGTATCCATAGGCAGATTCGACAACCCT +TTTATTAAAAATACTCTACATCAGCCGGCCATGGTGGCTCACGCCTGTAATCCCAGGACTTTGGGAGGCA +CAGGCAGGCAGATCACCTGAG + diff --git a/tests/data/otud4-review/README.md b/tests/data/otud4-review/README.md new file mode 100644 index 0000000..f4b9470 --- /dev/null +++ b/tests/data/otud4-review/README.md @@ -0,0 +1,13 @@ +# OTUD4 review, 2026-10-05 + +The call excerpts are from public CeGaT **normal-DNA** indel files. The VCF +contains GRCh37 chr4:146077122 TCAG>T (7 ALT / 323 depth); its transcript +annotation is NM_001102653.1:c.458_460delCTG p.A153del, flagged FP/HET. +The full RefSeq windows and Ensembl mapping place the same literal allele at +GRCh38 chr4:145155970. Full-source and excerpt SHA-256 values are separate in +`provenance.json`. + +This verifies a matching public allele for the newly populated source entry. +It does not verify the unavailable original Natera report, confirm a somatic +mutation, or turn the normal-DNA call into tumor evidence. Historical unresolved +entries keep their previous disposition. diff --git a/tests/data/otud4-review/cegat-normal-indels.annotated.tsv b/tests/data/otud4-review/cegat-normal-indels.annotated.tsv new file mode 100644 index 0000000..5a2b97b --- /dev/null +++ b/tests/data/otud4-review/cegat-normal-indels.annotated.tsv @@ -0,0 +1,2 @@ +chr4 146077122 TCAG T FP/HET 323 7 316 0.0217 OTUD4 CCDS47139.1;ENST00000454497;NM_001102653.1 inframe c.458_460delCTG p.A153del gCTGatgtg/gatgtg 0.0011363636363636363 SAS rs150581210 +chr4 146077122 TCAG T FP/HET 323 7 316 0.0217 OTUD4 CCDS93644.1;ENST00000447906.8;NM_001366057.1 inframe c.653_655delCTG p.A218del gCTGatgtg/gatgtg 0.0011363636363636363 SAS rs150581210 diff --git a/tests/data/otud4-review/cegat-normal-indels.vcf b/tests/data/otud4-review/cegat-normal-indels.vcf new file mode 100644 index 0000000..0761294 --- /dev/null +++ b/tests/data/otud4-review/cegat-normal-indels.vcf @@ -0,0 +1 @@ +chr4 146077122 . TCAG T . . DP=323;HET=0;HOM=1;RD=316;AD=7;AF=0.0217 GT 0/0 diff --git a/tests/data/otud4-review/mapping.json b/tests/data/otud4-review/mapping.json new file mode 100644 index 0000000..6e29a7d --- /dev/null +++ b/tests/data/otud4-review/mapping.json @@ -0,0 +1 @@ +{"mappings":[{"original":{"end":146077125,"strand":1,"start":146077122,"coord_system":"chromosome","assembly":"GRCh37","seq_region_name":"4"},"mapped":{"start":145155970,"end":145155973,"strand":1,"assembly":"GRCh38","seq_region_name":"4","coord_system":"chromosome"}}]} \ No newline at end of file diff --git a/tests/data/otud4-review/provenance.json b/tests/data/otud4-review/provenance.json new file mode 100644 index 0000000..4149411 --- /dev/null +++ b/tests/data/otud4-review/provenance.json @@ -0,0 +1,38 @@ +{ + "cegat-normal-indels.vcf": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/cegat/P116686_1_S000048/P116686_1_indels.vcf", + "sha256": "4e37a807db95fdc82b36e0512f8bc3f58a021bdd8268d3e59fde5fd9a8d29683", + "fixture_sha256": "199d8011e4c5974590545f8328d07a62a5260157aecb6f5689f3d575debeec68", + "selection": "OTUD4 chr4:146077122 call rows" + }, + "cegat-normal-indels.annotated.tsv": { + "url": "https://sid-sijbrandij-osteosarc-dataset.s3.us-west-2.amazonaws.com/vendor/cegat/P116686_1_S000048/P116686_1_indels.annotated.tsv", + "sha256": "18650e9b0cd000a8c7b58b3a562f7a621a8d61ff9c9deb389428ba542d99215e", + "fixture_sha256": "8493a851d6d0a2fac2daf00c77e49090cde68b339f53b6cd7ffc9133c4332d2d", + "selection": "OTUD4 chr4:146077122 call rows" + }, + "NC_000004.11.fasta": { + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000004.11&seq_start=146076622&seq_stop=146077622&rettype=fasta&retmode=text", + "sha256": "564dd3888f5ae4ed9b5248cc8ab41c52d18b1e90c12cbe15f2999698a923d999", + "fixture_sha256": "564dd3888f5ae4ed9b5248cc8ab41c52d18b1e90c12cbe15f2999698a923d999", + "selection": "whole reference window" + }, + "NC_000004.12.fasta": { + "url": "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nuccore&id=NC_000004.12&seq_start=145155470&seq_stop=145156470&rettype=fasta&retmode=text", + "sha256": "7563c0185bb92d5e129b4c679741b0a686d0b7d61e49ae87124edbde96be8e78", + "fixture_sha256": "7563c0185bb92d5e129b4c679741b0a686d0b7d61e49ae87124edbde96be8e78", + "selection": "whole reference window" + }, + "mapping.json": { + "url": "https://rest.ensembl.org/map/human/GRCh37/4:146077122..146077125:1/GRCh38?content-type=application/json", + "sha256": "e72fb653a74d8a092177dc6ad85135b40028e9187b7e90e0888f378ffc0b5087", + "fixture_sha256": "e72fb653a74d8a092177dc6ad85135b40028e9187b7e90e0888f378ffc0b5087", + "selection": "whole mapping response" + }, + "source-label.json": { + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/bam-metadata-consolidated.tsv", + "sha256": "cb620face7a055283818eea668a18695d12dc280e0ca7666cb637d37938da2ef", + "fixture_sha256": "74f566322b537d63e5e08ff0cc803b5132ac68daf220cc48f253d0948e6a1179", + "selection": "CeGaT P116686_1 source metadata row" + } +} diff --git a/tests/data/otud4-review/source-label.json b/tests/data/otud4-review/source-label.json new file mode 100644 index 0000000..16c04f9 --- /dev/null +++ b/tests/data/otud4-review/source-label.json @@ -0,0 +1,31 @@ +{ + "receipt": { + "etag": "\"b288e082de0413ca220ef852e48c382d\"", + "filename": "bam-metadata-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-05T14:27:34.206463+00:00", + "sha256": "cb620face7a055283818eea668a18695d12dc280e0ca7666cb637d37938da2ef", + "size": 30927, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/bam-metadata-consolidated.tsv" + }, + "row": { + "display_name": "T0 CeGaT Blood WES 2022-12 (P116686_1 vendor)", + "tissue": "Blood", + "assay": "WES", + "timepoint": "T0", + "sample_date": "2022-12", + "provider": "CeGaT", + "analyst": "CeGaT", + "b2_path": "", + "s3_path": "vendor/cegat/P116686_1_S000048/P116686_1.bam", + "b2_size_bytes": "", + "b2_size_human": "", + "s3_size_bytes": "10108589764", + "s3_size_human": "9.41 GiB", + "md5sum": "eeb686e3c605213be6c0c8ac7348f2d0", + "is_primary": "", + "is_superseded_by": "", + "notes": "CeGaT normal DNA (exome kit S000048, hg19, chrY PAR masked). Sequenced 2023-03-22 on the T0 material collected 2022-12-16. Source data for the CeGaT peptide neoantigen vaccine." + } +} diff --git a/tests/data/sample-locations-2026-10-07.json b/tests/data/sample-locations-2026-10-07.json new file mode 100644 index 0000000..71b8fe4 --- /dev/null +++ b/tests/data/sample-locations-2026-10-07.json @@ -0,0 +1,1082 @@ +{ + "accessed": "2026-10-07", + "selection": "Complete FASTQ rows for eleven filename-prefix and Tempus alias locations, their parent-directory inventory entries, and the complete sample registry. Parsed source excerpts; original values retained.", + "receipts": { + "fastqs": { + "etag": "\"ddcf14af8eeb0138c561edfa83ba653a\"", + "filename": "fastqs-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-07T16:17:10.649983+00:00", + "sha256": "4e7aed516e48f448262370eb23103603f10cae69296c920a2e60e6727c10b762", + "size": 75106, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/fastqs-consolidated.tsv" + }, + "bucket": { + "etag": "\"dlx77r2uc9hdqnr7f\"", + "filename": "bucket_listing.json", + "last_modified": "Mon, 05 Oct 2026 20:48:31 GMT", + "md5": null, + "retrieved_at": "2026-10-07T16:17:03.063456+00:00", + "sha256": "3567118cd3e3ee6dd5533916da86079e40028427db92ba150c8ee7ee1d0d8d12", + "size": 44778363, + "url": "https://osteosarc.com/bucket_listing.json" + }, + "specimens": { + "etag": "\"65a4a32376236a8d4dbee439f79fd8ed\"", + "filename": "samples-consolidated.tsv", + "last_modified": null, + "md5": null, + "retrieved_at": "2026-10-07T16:17:09.335541+00:00", + "sha256": "245e78a7ff1bd9f9713cd1e356c24cc803c2f3d0cfdce88da6e043791547029d", + "size": 17386, + "url": "https://gitlab.com/slowkow/osteosarc.com/-/raw/main/scripts/data/samples-consolidated.tsv" + }, + "data_page": { + "etag": null, + "filename": "data", + "last_modified": "Mon, 05 Oct 2026 20:48:40 GMT", + "md5": null, + "retrieved_at": "2026-10-07T16:17:07.335081+00:00", + "sha256": "6f6b1bd9f3603ed3ed00b4582bd0792e752f9992f30f4baeacf7856f4b174866", + "size": 131895, + "url": "https://osteosarc.com/data/" + } + }, + "fastqs": [ + { + "display_name": "T0 CeGaT Tumor RNA 2022-12 (P116686_3 vendor)", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_tumor", + "provider": "CeGaT", + "analyst": "", + "assay": "RNA", + "b2_folder": "", + "s3_folder": "vendor/cegat/P116686_2_S000048/P116686_3", + "file_count": "2", + "total_size_bytes": "7210584452", + "total_size_human": "6.72 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "CeGaT tumor RNA (STAR 2.7.3a, hg19). Sequenced 2023-03-22 on the T0 material collected 2022-12-16. Files live under P116686_2_S000048/ despite the P116686_3 sample id." + }, + { + "display_name": "T0 CeGaT Tumor WES 2022-12 (P116686_2 vendor)", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_tumor", + "provider": "CeGaT", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/cegat/P116686_2_S000048/P116686_2", + "file_count": "2", + "total_size_bytes": "13089275642", + "total_size_human": "12.19 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "CeGaT tumor DNA (exome kit S000048, hg19). Sequenced 2023-03-22 on the T0 material collected 2022-12-16. Ships _somatic.tsv; source data for the CeGaT neoantigen vaccine." + }, + { + "display_name": "T0 Natera Tumor WES 2022-12 (Ashion I006353-01D-01L)", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_tumor", + "provider": "Natera", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/natera/fastq/28578842.1-2-FXG_C069-018634_WF00085349_I006353-01D-01L_TumorDNA", + "file_count": "2", + "total_size_bytes": "29164818628", + "total_size_human": "27.16 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Natera/Ashion hybrid-capture tumour DNA (WES-like; @RG CN:ASHION, LB I006353-01D-01L, SM ..._TumorDNA). Case 9759206 (Altera WES) or 8592254 (Signatera WES): the delivery does not say which. GRCh37 hs37d5, contigs 1, 2, ... (no chr prefix). Not as delivered: FASTQ regenerated in this project from vendor/natera/28578842.1-2-FXG_C069-018634_WF00085349_I006353-01D-01L_TumorDNA.bam (samtools collate + fastq; secondary and supplementary alignments dropped, duplicates kept), 258,112,616 read pairs. Natera delivered no FASTQ." + }, + { + "display_name": "T0 Tempus Tumor WES 2022-12 (TL-24-ALMY2X4KMV vendor)", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_tumor", + "provider": "Tempus", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/T", + "file_count": "2", + "total_size_bytes": "15470759019", + "total_size_human": "14.41 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Tempus xE whole exome, tumour; GRCh37. Byte-identical to the catalog copy under rna-seq/tempus/TL-24-ALMY2X4KMV/WES/. Tumor and normal share one FastQ/ directory, so the row key appends /T." + }, + { + "display_name": "T0 Natera Tumor WGS 2022-12 (case 17849104 0198480a, tumour inferred)", + "tissue": "Tumor", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_tumor", + "provider": "Natera", + "analyst": "", + "assay": "WGS", + "b2_folder": "", + "s3_folder": "vendor/natera/fastq/0198480a-d9d5-7fe4-baed-4d696baf665b", + "file_count": "2", + "total_size_bytes": "317798412631", + "total_size_human": "295.97 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Natera WGS, case 17849104; GRCh37 hs37d5, contigs chr1, chr2, ... Tumour is INFERRED, not vendor-confirmed: the BAM's @RG SM is only its UUID; at heterozygous sites only 8% of allele fractions sit near 0.5 (they pile up at 0.20-0.25), and it is ~123x deep against ~45x for the 0196b23b normal, on a separate flowcell. Confirm before somatic use. Not as delivered: FASTQ regenerated in this project from vendor/natera/0198480a-d9d5-7fe4-baed-4d696baf665b.bam (samtools collate + fastq; secondary and supplementary alignments dropped, duplicates kept), 1,447,357,058 read pairs. Natera delivered no FASTQ." + }, + { + "display_name": "T0 CeGaT Blood WES 2022-12 (P116686_1 vendor)", + "tissue": "Blood", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_blood", + "provider": "CeGaT", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/cegat/P116686_1_S000048/P116686_1", + "file_count": "2", + "total_size_bytes": "11321968597", + "total_size_human": "10.54 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "CeGaT normal DNA (exome kit S000048, hg19, chrY PAR masked). Sequenced 2023-03-22 on the T0 material collected 2022-12-16. Source data for the CeGaT peptide neoantigen vaccine." + }, + { + "display_name": "T0 Natera Blood WES 2022-12 (Ashion i005199-01D-01L)", + "tissue": "Blood", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_blood", + "provider": "Natera", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/natera/fastq/28578183.3-2-BXE_C069-018634_WF00085349_i005199-01D-01L_GermlineDNA", + "file_count": "2", + "total_size_bytes": "14016415816", + "total_size_human": "13.05 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Natera/Ashion hybrid-capture germline DNA (WES-like; @RG CN:ASHION, LB i005199-01D-01L, SM ..._GermlineDNA), the normal for the Ashion tumour; its source and draw date are not recorded, so it is filed with T0_blood. Case 9759206 (Altera WES) or 8592254 (Signatera WES): the delivery does not say which. GRCh37 hs37d5, contigs 1, 2, ... (no chr prefix). Not as delivered: FASTQ regenerated in this project from vendor/natera/28578183.3-2-BXE_C069-018634_WF00085349_i005199-01D-01L_GermlineDNA.bam (samtools collate + fastq; secondary and supplementary alignments dropped, duplicates kept), 123,411,122 read pairs. Natera delivered no FASTQ." + }, + { + "display_name": "T0 Natera Blood WGS 2022-12 (case 17849104 0196b23b, normal inferred)", + "tissue": "Blood", + "timepoint": "T0", + "sample_date": "2022-12", + "sample_id": "T0_blood", + "provider": "Natera", + "analyst": "", + "assay": "WGS", + "b2_folder": "", + "s3_folder": "vendor/natera/fastq/0196b23b-c139-7fb0-91ff-65f19670d743", + "file_count": "2", + "total_size_bytes": "113447783544", + "total_size_human": "105.66 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Natera WGS, case 17849104: the germline normal, INFERRED, not vendor-confirmed (the BAM's @RG SM is only its UUID; allele fractions at heterozygous sites peak at 0.45-0.50, ~45x). Whether it is blood or adjacent normal tissue, and when it was collected, is not recorded; filed with T0_blood. GRCh37 hs37d5, contigs chr1, chr2, ... Confirm before somatic use. Not as delivered: FASTQ regenerated in this project from vendor/natera/0196b23b-c139-7fb0-91ff-65f19670d743.bam (samtools collate + fastq; secondary and supplementary alignments dropped, duplicates kept), 494,239,896 read pairs. Natera delivered no FASTQ." + }, + { + "display_name": "T1 UCLA Tumor RNA 2024-06 (SARC0277-100K vendor)", + "tissue": "Tumor", + "timepoint": "T1", + "sample_date": "2024-06", + "sample_id": "T1_tumor", + "provider": "UCLA", + "analyst": "", + "assay": "RNA", + "b2_folder": "", + "s3_folder": "vendor/ucla/SARC0277-T1-2024/SARC0277-100K", + "file_count": "2", + "total_size_bytes": "9044024289", + "total_size_human": "8.42 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "UCLA T1 tumour RNA-seq from the 2024-06-06 CT-guided biopsy (PREMOST SARC0277; same UCLA sample name and flowcell lane 227V5CLT4:6 as the T1 WGS). The patient (somalier run 5; 0.7% foreign alleles). Untrimmed 2x150: trim adapters before aligning. Reprocessed as rna-seq/reprocessed/SARC0277_T1/." + }, + { + "display_name": "T2 UCLA Organoid WGS 2025-01 (SARC0277_3 ORGANOID vendor)", + "tissue": "Organoid", + "timepoint": "T2", + "sample_date": "2025-01", + "sample_id": "T2_organoid", + "provider": "UCLA", + "analyst": "", + "assay": "WGS", + "b2_folder": "", + "s3_folder": "vendor/ucla/SARC0277-T2-2025/16-SARC0277_3_ORGANOID_DNA", + "file_count": "2", + "total_size_bytes": "237273707752", + "total_size_human": "220.98 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "UCLA T2 organoid WGS (organoid of the 2025-01-28 T2 biopsy SARC0277_3; library dated 02-02-25). The patient (somalier IBS0 0; PURPLE contamination 0). About 1.3% mouse reads (culture matrix), adapter dimers and poly-G tails: trim and remove mouse reads before variant calling." + }, + { + "display_name": "Tempus Blood WES 2024-09-21 (TL-24-ALMY2X4KMV vendor)", + "tissue": "Blood", + "timepoint": "", + "sample_date": "2024-09-21", + "sample_id": "blood_2024-09-21", + "provider": "Tempus", + "analyst": "", + "assay": "WES", + "b2_folder": "", + "s3_folder": "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/N", + "file_count": "2", + "total_size_bytes": "9202603169", + "total_size_human": "8.57 GiB", + "is_primary": "", + "is_superseded_by": "", + "notes": "Germline normal for the Tempus xE/xT tests of the T0 tumour; blood drawn 2024-09-21. GRCh37. Byte-identical to the catalog copy under rna-seq/tempus/TL-24-ALMY2X4KMV/WES/. Tumor and normal share one FastQ/ directory, so the row key appends /N." + } + ], + "bucket": [ + [ + "vendor/cegat/P116686_1_S000048/P116686_1.1.fastq.gz", + 5570909383, + 1784584087 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1.2.fastq.gz", + 5751059214, + 1784584168 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1.bai", + 7056368, + 1784584305 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1.bam", + 10108589764, + 1784584305 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_cnvs.tsv", + 102888, + 1784584305 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_indels.annotated.tsv", + 7789915, + 1784584305 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_indels.vcf", + 1387823, + 1784584306 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_snvs.annotated.tsv", + 18428794, + 1784584306 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_snvs.vcf", + 3936796, + 1784584307 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_snvs_indels.annotated.tsv", + 26218543, + 1784584307 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_snvs_indels.vcf", + 5323765, + 1784584312 + ], + [ + "vendor/cegat/P116686_1_S000048/P116686_1_versions.txt", + 265, + 1784584313 + ], + [ + "vendor/cegat/P116686_1_S000048/S000048_hg19_targets.bed", + 8759608, + 1784584314 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2.1.fastq.gz", + 6401770586, + 1784584317 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2.2.fastq.gz", + 6687505056, + 1784584319 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2.bai", + 7150296, + 1784584389 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2.bam", + 12018624310, + 1784584390 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_cnvs.tsv", + 1629973, + 1784584632 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_indels.annotated.tsv", + 6748887, + 1784584632 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_indels.vcf", + 1218335, + 1784584633 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_snvs.annotated.tsv", + 18397314, + 1784584634 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_snvs.vcf", + 3924486, + 1784584639 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_snvs_indels.annotated.tsv", + 25146035, + 1784584640 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_snvs_indels.vcf", + 5141967, + 1784584645 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_somatic.tsv", + 215311, + 1784584646 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_2_versions.txt", + 265, + 1784584647 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_3.1.fastq.gz", + 3641438230, + 1784584648 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_3.2.fastq.gz", + 3569146222, + 1784585146 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_3.bai", + 7515296, + 1784585192 + ], + [ + "vendor/cegat/P116686_2_S000048/P116686_3.bam", + 7823924680, + 1784585192 + ], + [ + "vendor/cegat/P116686_2_S000048/S000048_hg19_targets.bed", + 8759608, + 1784585243 + ], + [ + "vendor/natera/fastq/0196b23b-c139-7fb0-91ff-65f19670d743_R1.fastq.gz", + 56574211354, + 1785600471 + ], + [ + "vendor/natera/fastq/0196b23b-c139-7fb0-91ff-65f19670d743_R2.fastq.gz", + 56873572190, + 1785600471 + ], + [ + "vendor/natera/fastq/0198480a-d9d5-7fe4-baed-4d696baf665b_R1.fastq.gz", + 159125815454, + 1785601069 + ], + [ + "vendor/natera/fastq/0198480a-d9d5-7fe4-baed-4d696baf665b_R2.fastq.gz", + 158672597177, + 1785601636 + ], + [ + "vendor/natera/fastq/28578183.3-2-BXE_C069-018634_WF00085349_i005199-01D-01L_GermlineDNA_R1.fastq.gz", + 6903663785, + 1785604082 + ], + [ + "vendor/natera/fastq/28578183.3-2-BXE_C069-018634_WF00085349_i005199-01D-01L_GermlineDNA_R2.fastq.gz", + 7112752031, + 1785604098 + ], + [ + "vendor/natera/fastq/28578842.1-2-FXG_C069-018634_WF00085349_I006353-01D-01L_TumorDNA_R1.fastq.gz", + 14335194230, + 1785604525 + ], + [ + "vendor/natera/fastq/28578842.1-2-FXG_C069-018634_WF00085349_I006353-01D-01L_TumorDNA_R2.fastq.gz", + 14829624398, + 1785604560 + ], + [ + "vendor/natera/fastq/readcounts.tsv", + 328, + 1785605427 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV.tar.gz", + 24680194141, + 1784586025 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV.tar.gz.md5", + 58, + 1784586025 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_N_DSQ2_1.fastq.gz", + 4491858339, + 1784586026 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_N_DSQ2_1.fastq.gz.seqkit-stats.tsv", + 337, + 1784586511 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_N_DSQ2_2.fastq.gz", + 4710744830, + 1784586512 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_N_DSQ2_2.fastq.gz.seqkit-stats.tsv", + 337, + 1784586575 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_T_DSQ2_1.fastq.gz", + 7693459651, + 1784586576 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_T_DSQ2_1.fastq.gz.seqkit-stats.tsv", + 338, + 1784586648 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_T_DSQ2_2.fastq.gz", + 7777299368, + 1784586648 + ], + [ + "vendor/tempus/TL-24-ALMY2X4KMV/DNA/FastQ/TL-24-ALMY2X4KMV_T_DSQ2_2.fastq.gz.seqkit-stats.tsv", + 339, + 1784586674 + ], + [ + "vendor/ucla/SARC0277-T1-2024/SARC0277-100K-DNA-RNA_S1_L006_R1_001.fastq.gz", + 4535659300, + 1790352617 + ], + [ + "vendor/ucla/SARC0277-T1-2024/SARC0277-100K-DNA-RNA_S1_L006_R2_001.fastq.gz", + 4508364989, + 1790352616 + ], + [ + "vendor/ucla/SARC0277-T1-2024/SARC0277-ORGANOID-DNA-RNA_S2_L006_R1_001.fastq.gz", + 4936758213, + 1790352617 + ], + [ + "vendor/ucla/SARC0277-T1-2024/SARC0277-ORGANOID-DNA-RNA_S2_L006_R2_001.fastq.gz", + 4780370521, + 1790352617 + ], + [ + "vendor/ucla/SARC0277-T2-2025/16-SARC0277_3_ORGANOID_DNA_02-02-25_S17_L006_R1_001.fastq.gz", + 118389471680, + 1790352617 + ], + [ + "vendor/ucla/SARC0277-T2-2025/16-SARC0277_3_ORGANOID_DNA_02-02-25_S17_L006_R2_001.fastq.gz", + 118884236072, + 1790352690 + ], + [ + "vendor/ucla/SARC0277-T2-2025/49_SARC0277_3-organoid-RNA-02-02-25_S133_L007_R1_001.fastq.gz", + 3905871836, + 1790356698 + ], + [ + "vendor/ucla/SARC0277-T2-2025/49_SARC0277_3-organoid-RNA-02-02-25_S133_L007_R2_001.fastq.gz", + 4009037618, + 1790356513 + ] + ], + "specimens": [ + { + "sample_id": "T0_tumor", + "display_name": "T0 Tumor 2022-12 (diagnosis resection)", + "tissue": "Tumor", + "timepoint": "T0", + "collection_date": "2022-12-16", + "collection_site": "UCSF", + "tissue_source": "Primary tumor resection", + "vendors_involved": "BostonGene; CeGaT; Personalis; Tempus", + "assays_run": "Panel; RNA; WES; WGS", + "bam_count": "17", + "bam_display_names": "T0 Personalis Tumor DRAGEN; T0 Personalis Tumor WGS 2022-12; T0 Personalis Tumor oncoanalyser; T0 BostonGene Tumor WES 2022-12; T0 BostonGene Tumor WES 2022-12 (BG003082 vendor); T0 CeGaT Tumor WES 2022-12 (P116686_2 vendor); T0 Tempus Tumor WES 2022-12 (TL-24-ALMY2X4KMV vendor); T0 Tempus Tumor Panel 2022-12 (TL-24-5GQLV9WSXQ xT 648-gene vendor); T0 BostonGene Tumor RNA 2022-12 (BG003082 reprocessed); T0 BostonGene Tumor RNA 2022-12 (BG003082 vendor); T0 BostonGene Tumor RNA oncoanalyser; T0 CeGaT Tumor RNA 2022-12 (P116686_3 reprocessed); T0 CeGaT Tumor RNA 2022-12 (P116686_3 vendor); T0 Personalis Tumor RNA 2022-12 (DRF-PEN28653_WTS reprocessed); T0 Tempus Tumor RNA 2022-12 (TL-24-ALMY2X4KMV vendor); T0 Tempus Tumor RNA 2022-12 (TL-24-KCVBE1UI1P reprocessed); T0 Tempus Tumor RNA 2022-12 (TL-24-KCVBE1UI1P vendor)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Initial diagnosis; assayed by BostonGene (WES, RNA), Personalis (WGS; WES and WTS re-delivered 2026-08 as DRF-PEN28653, replacing the first Personalis RNA library, which was another person's) and CeGaT (WES, RNA; sequenced 2023-03-22); also Tempus xT/xE/xR on block S22-38225 B12 (tested 2024-09/10)" + }, + { + "sample_id": "T0_blood", + "display_name": "T0 Blood 2022-12 (germline normal)", + "tissue": "Blood", + "timepoint": "T0", + "collection_date": "2022-12-16", + "collection_site": "UCSF", + "tissue_source": "Whole blood", + "vendors_involved": "BostonGene; CeGaT", + "assays_run": "WES", + "bam_count": "5", + "bam_display_names": "T0 BostonGene Blood WES 2022-12; T0 BostonGene Blood WES 2022-12 (BG003082 vendor); T0 BostonGene Blood WES 2022-12 (BG009368 re-delivery, Sarek 2024 run); T0 BostonGene Blood WES 2022-12 (BG009368 vendor, re-delivered T0 normal); T0 CeGaT Blood WES 2022-12 (P116686_1 vendor)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Germline reference for variant calling. Composite of separate draws per the vendor reports: BostonGene normal whole blood collected 2023-01-27 (re-delivered with the 2024 BG009368 order). The Personalis normal (collected 2024-06-11) is blood_2024-06-11" + }, + { + "sample_id": "T1_tumor", + "display_name": "T1 Tumor 2024-06 (recurrence biopsy)", + "tissue": "Tumor", + "timepoint": "T1", + "collection_date": "2024-06-06", + "collection_site": "UCLA", + "tissue_source": "Recurrent tumor biopsy (CT-guided, T4-T5)", + "vendors_involved": "BostonGene; UCLA; UCSF", + "assays_run": "PacBio; RNA; WES; WGS; scRNA; scRNA_ONT", + "bam_count": "12", + "bam_display_names": "T1 UCLA Tumor DRAGEN; T1 UCLA Tumor WGS 2024-06; T1 UCLA Tumor oncoanalyser; T1 BostonGene Tumor WES 2024-06; T1 BostonGene Tumor WES 2024-06 (BG009368 vendor); T1 BostonGene Tumor RNA 2024-06 (BG009368 reprocessed); T1 BostonGene Tumor RNA 2024-06 (BG009368 vendor); T1 BostonGene Tumor RNA oncoanalyser; T1 UCLA Tumor RNA 2024-06 (SARC0277_T1 reprocessed); T1 UCSF Tumor scRNA 2024-06 (Kamil CellRanger); T1 UCSF Tumor PacBio 2024-06; T1 UCSF Tumor ONT 2024-06", + "is_primary": "", + "is_superseded_by": "", + "notes": "Recurrence; BostonGene (WES, RNA) + UCLA (WGS, RNA) + UCSF scRNA/ONT/PacBio. UCLA WGS/organoid and UCLA RNA (SARC0277-100K, reprocessed as SARC0277_T1) from the 2024-06-06 CT-guided biopsy (PREMOST SARC0277); BostonGene BG009368 WES/RNA from the 2024-06-11 CT-guided biopsy (UCLA SSU-24-07964); source procedure of the UCSF scRNA/ONT/PacBio unconfirmed (timeline says 2024-06-06; a UCSF CT biopsy on 2024-05-10 also exists)" + }, + { + "sample_id": "T1_blood", + "display_name": "T1 Blood 2024-06 (germline normal)", + "tissue": "Blood", + "timepoint": "T1", + "collection_date": "2024-06-06", + "collection_site": "UCLA", + "tissue_source": "Whole blood", + "vendors_involved": "UCLA", + "assays_run": "WGS", + "bam_count": "7", + "bam_display_names": "T1 UCLA Blood DRAGEN; T1 UCLA Blood DRAGEN (T2 pair); T1 UCLA Blood DRAGEN (organoid pair); T1 UCLA Blood WGS 2024-06; UCLA Blood oncoanalyser; UCLA Blood oncoanalyser (T1 organoid pair); UCLA Blood oncoanalyser (T2 pair)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Composite: UCLA WGS normal drawn 2024-06-06 (T1 WGS Normal coverage ~60x; the DRAGEN T2-pair and organoid-pair normals and the oncoanalyser T2-pair and T1-organoid-pair normals are re-alignments of this same library). The BostonGene WES normal is the reused 2023-01-27 draw (see T0_blood), the Personalis normal (drawn 2024-06-11) is blood_2024-06-11 and the Tempus normal is blood_2024-09-21" + }, + { + "sample_id": "T1_organoid", + "display_name": "T1 Organoid 2024-06", + "tissue": "Organoid", + "timepoint": "T1", + "collection_date": "2024-06-06", + "collection_site": "UCLA", + "tissue_source": "Patient-derived organoid from T1 tumor (PREMOST, Soragni lab, UCLA)", + "vendors_involved": "UCLA", + "assays_run": "WGS", + "bam_count": "3", + "bam_display_names": "T1 organoid Tumor DRAGEN; T1 organoid Tumor oncoanalyser; T1 UCLA Organoid WGS 2024-06", + "is_primary": "", + "is_superseded_by": "", + "notes": "WGS only; from the 2024-06-06 biopsy. Sarek-processed by Jeremiah/Alfredo; also DRAGEN and oncoanalyser re-alignments" + }, + { + "sample_id": "T2_tumor", + "display_name": "T2 Tumor 2025-01 (biopsy)", + "tissue": "Tumor", + "timepoint": "T2", + "collection_date": "2025-01-28", + "collection_site": "UCLA", + "tissue_source": "Tumor biopsy (CT-guided, T4-T5)", + "vendors_involved": "UCLA; UCSF", + "assays_run": "RNA; WGS; scRNA; scRNA_ONT", + "bam_count": "8", + "bam_display_names": "T2 UCLA Tumor DRAGEN; T2 UCLA Tumor WGS 2025-01; T2 UCLA Tumor oncoanalyser; T2 UCLA Tumor RNA 2025-01 (SARC0277 reprocessed); T2 UCLA Tumor RNA 2025-01 (protocol-aligned); T2 UCLA Tumor RNA oncoanalyser; T2 UCSF Tumor scRNA 2025-01 (Kamil CellRanger); T2 UCSF Tumor ONT 2025-01", + "is_primary": "", + "is_superseded_by": "", + "notes": "UCLA WGS (cleaned, 130.97x coverage); UCLA RNA; UCSF scRNA/ONT run on the UCLA specimen. The bucket folder genomics-bulk/2025.01.06 is a delivery label, not the collection date" + }, + { + "sample_id": "T2_organoid", + "display_name": "T2 Organoid 2025-01", + "tissue": "Organoid", + "timepoint": "T2", + "collection_date": "2025-01-28", + "collection_site": "UCLA", + "tissue_source": "Patient-derived organoid from T2 tumor (PREMOST, UCLA)", + "vendors_involved": "UCLA", + "assays_run": "WGS", + "bam_count": "1", + "bam_display_names": "T2 UCLA Organoid oncoanalyser", + "is_primary": "", + "is_superseded_by": "", + "notes": "From the 2025-01-28 biopsy (UCLA SARC0277_3). UCLA WGS delivered 2026-09 (library dated 02-02-25): the patient, with ~1.3% mouse reads. Its RNA-seq from the same delivery is about a quarter another person's and is excluded" + }, + { + "sample_id": "T3_tumor", + "display_name": "T3 Tumor 2025-04 (resection)", + "tissue": "Tumor", + "timepoint": "T3", + "collection_date": "2025-04-17", + "collection_site": "MSKCC", + "tissue_source": "Tumor resection (T4-T5)", + "vendors_involved": "UCSF", + "assays_run": "scRNA; scRNA_ONT", + "bam_count": "2", + "bam_display_names": "T3 UCSF Tumor scRNA 2025-04 (Kamil CellRanger); T3 UCSF Tumor ONT 2025-04", + "is_primary": "", + "is_superseded_by": "", + "notes": "No bulk DNA/RNA; scRNA + ONT only (processed at UCSF)" + }, + { + "sample_id": "T3_tumor_CD45neg", + "display_name": "T3 Tumor 2025-04 (CD45- enriched)", + "tissue": "Tumor", + "timepoint": "T3", + "collection_date": "2025-04-17", + "collection_site": "MSKCC", + "tissue_source": "CD45-negative enriched fraction of T3 tumor", + "vendors_involved": "UCSF", + "assays_run": "scRNA", + "bam_count": "1", + "bam_display_names": "T3 UCSF Tumor scRNA 2025-04 (CD45neg)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Sorted fraction of same biological sample as T3_tumor" + }, + { + "sample_id": "blood_2024-06-11", + "display_name": "T1 Blood 2024-06-11 (Personalis germline normal)", + "tissue": "Blood", + "timepoint": "T1", + "collection_date": "2024-06-11", + "collection_site": "", + "tissue_source": "Whole blood", + "vendors_involved": "Personalis", + "assays_run": "WGS", + "bam_count": "3", + "bam_display_names": "T1 Personalis Blood DRAGEN; T1 Personalis Blood WGS 2024-06; T1 Personalis Blood oncoanalyser", + "is_primary": "", + "is_superseded_by": "", + "notes": "Matched germline normal for the Personalis tests of the T0 tumour: NeXT Personal WGS (normal PSNLDx240603252) and NeXT Dx (accession PEN28653; its WES normal re-delivered 2026-08 as DRF-PEN28653); drawn 2024-06-11, received by Personalis 2024-06-12 (NeXT Dx report PEN28653). Filed as T1: the same day as the BostonGene T1 biopsy and five days after the UCLA T1 draw (T1_blood). Draw site not documented" + }, + { + "sample_id": "blood_2024-09-21", + "display_name": "Blood 2024-09-21 (Tempus germline normal)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2024-09-21", + "collection_site": "", + "tissue_source": "Whole blood", + "vendors_involved": "Tempus", + "assays_run": "WES", + "bam_count": "1", + "bam_display_names": "Tempus Blood WES 2024-09-21 (TL-24-ALMY2X4KMV vendor)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Matched germline normal for the Tempus xE/xT tests (TL-24-ALMY2X4KMV, TL-24-5GQLV9WSXQ) of the T0 tumour; drawn 2024-09-21, received by Tempus 2024-09-23 (Tempus reports). Draw site not documented" + }, + { + "sample_id": "blood_2025-01-25", + "display_name": "Longitudinal Blood 2025-01-25 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-01-25", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "scRNA", + "bam_count": "3", + "bam_display_names": "Hudson Lab Blood scRNA 2025-01 (Pool 1); Hudson Lab Blood scRNA 2025-01 (Pool 2); Hudson Lab Blood scRNA 2025-01 (Pool 3)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq (GEX), 3 pools; Draw site not documented; PBMCs cryopreserved by Pathfinder Oncology and shipped to the Hudson Lab (Baylor)" + }, + { + "sample_id": "blood_2025-04-09", + "display_name": "Longitudinal Blood 2025-04-09 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-04-09", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "scRNA", + "bam_count": "5", + "bam_display_names": "Hudson Lab Blood scRNA 2025-04 (Pool 1); Hudson Lab Blood scRNA 2025-04 (Pool 2); Hudson Lab Blood scRNA 2025-04 (Pool 3); Hudson Lab Blood scRNA 2025-04 (Pool 4); Hudson Lab Blood scRNA 2025-04 (Pool 5)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq (GEX), 5 pools; Draw site not documented; PBMCs cryopreserved by Pathfinder Oncology and shipped to the Hudson Lab (Baylor)" + }, + { + "sample_id": "blood_2025-05-01", + "display_name": "Longitudinal Blood 2025-05-01 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-05-01", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "4", + "bam_display_names": "Hudson Lab Blood CITE 2025-05 (Pool 1); Hudson Lab Blood CITE 2025-05 (Pool 2); Hudson Lab Blood CITE 2025-05 (Pool 3); Hudson Lab Blood CITE 2025-05 (Pool 4)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 4 pools; Draw site not documented; PBMCs cryopreserved at UCSF and shipped to the Hudson Lab (Baylor); 2025-05-01 is a hand-entered date override (no day in the run folder); draw likely 2025-05-06 per flow workbook (SRC044-B4)/MRD/labs (unconfirmed)" + }, + { + "sample_id": "blood_2025-06-26", + "display_name": "Longitudinal Blood 2025-06-26 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-06-26", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "4", + "bam_display_names": "Hudson Lab Blood CITE 2025-06 (Pool 1); Hudson Lab Blood CITE 2025-06 (Pool 2); Hudson Lab Blood CITE 2025-06 (Pool 3); Hudson Lab Blood CITE 2025-06 (Pool 4)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 4 pools; Draw site not documented; whole blood shipped to the Hudson Lab (Baylor), which isolated the PBMCs; run token 260625 = processing/capture date; draw likely 2025-06-25 per MRD/labs/timeline (flow workbook says 2025-06-24) (unconfirmed)" + }, + { + "sample_id": "blood_2025-07-24", + "display_name": "Longitudinal Blood 2025-07-24 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-07-24", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "3", + "bam_display_names": "Hudson Lab Blood CITE 2025-07 (Pool 1); Hudson Lab Blood CITE 2025-07 (Pool 2); Hudson Lab Blood CITE 2025-07 (Pool 3)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 3 pools; Draw site not documented; whole blood shipped to the Hudson Lab (Baylor), which isolated the PBMCs; run token 250724 = processing/capture date; draw likely 2025-07-22 per flow workbook/MRD/labs (unconfirmed)" + }, + { + "sample_id": "blood_2025-08-21", + "display_name": "Longitudinal Blood 2025-08-21 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-08-21", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "4", + "bam_display_names": "Hudson Lab Blood CITE 2025-08 (Pool 1); Hudson Lab Blood CITE 2025-08 (Pool 2); Hudson Lab Blood CITE 2025-08 (Pool 3); Hudson Lab Blood CITE 2025-08 (Pool 4)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 4 pools; Draw site not documented; whole blood shipped to the Hudson Lab (Baylor), which isolated the PBMCs; run token 250821 = processing/capture date; draw likely 2025-08-20 per flow workbook/MRD/labs (unconfirmed)" + }, + { + "sample_id": "blood_2025-09-22", + "display_name": "Longitudinal Blood 2025-09-22 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-09-22", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "1", + "bam_display_names": "Hudson Lab Blood CITE 2025-09", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 1 pool; Draw site not documented; whole blood shipped to the Hudson Lab (Baylor), which isolated the PBMCs; run token 250922 = processing/capture date; draw likely 2025-09-18 per flow workbook/MRD/labs (unconfirmed)" + }, + { + "sample_id": "blood_2025-11-06", + "display_name": "Longitudinal Blood 2025-11-06 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-11-06", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "6", + "bam_display_names": "Hudson Lab Blood CITE 2025-11 (Pool 1); Hudson Lab Blood CITE 2025-11 (Pool 2); Hudson Lab Blood CITE 2025-11 (capture1); Hudson Lab Blood CITE 2025-11 (capture2); Hudson Lab Blood CITE 2025-11 (capture3); Hudson Lab Blood CITE 2025-11 (capture4)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq; covers both Nov2025 and Nov2025v2 capture sets; Draw site not documented; whole blood shipped to the Hudson Lab (Baylor), which isolated the PBMCs; 2025-11-06 is a hand-entered date override; draw likely 2025-11-05 per flow workbook/MRD/labs (unconfirmed)" + }, + { + "sample_id": "blood_2025-12-20", + "display_name": "Longitudinal Blood 2025-12-20 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2025-12-20", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "Hudson Lab", + "assays_run": "CITE", + "bam_count": "4", + "bam_display_names": "Hudson Lab Blood CITE 2025-12 (capture1); Hudson Lab Blood CITE 2025-12 (capture2); Hudson Lab Blood CITE 2025-12 (capture3); Hudson Lab Blood CITE 2025-12 (capture4)", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab CITE-seq, 4 captures; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); 2025-12-20 is a hand-entered date override; the flow workbook dates the December draw 2025-12-29 and no lab draw is recorded on 2025-12-20 (unconfirmed)" + }, + { + "sample_id": "blood_2026-01-20", + "display_name": "Longitudinal Blood 2026-01-20 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-01-20", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 2 captures (capture1: GEX+TCR plus a shallow antibody-capture library; capture2: GEX+CITE+TCR); Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); 2026-01-20 is a hand-entered date override; draw likely 2026-01-23 per flow workbook/labs (unconfirmed); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-02-13", + "display_name": "Longitudinal Blood 2026-02-13 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-02-13", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-02-25", + "display_name": "Longitudinal Blood 2026-02-25 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-02-25", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 2 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); capture folder date; draw likely 2026-02-23 per flow workbook/labs (unconfirmed); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-03-09", + "display_name": "Longitudinal Blood 2026-03-09 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-03-09", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-03-24", + "display_name": "Longitudinal Blood 2026-03-24 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-03-24", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 4 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-04-16", + "display_name": "Longitudinal Blood 2026-04-16 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-04-16", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq: a first run (2026-04-16/capture1) and a re-run (2026-04-16_run1, 2 captures), each \u00d7 GEX+CITE+TCRab+TCRgd; the scRNA-seq analysis uses only the re-run (the first run gave 2,013 cells, the re-run 16,686); Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-04-30", + "display_name": "Longitudinal Blood 2026-04-30 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-04-30", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq: a first run (2026-04-30/capture1) and a re-run (2026-04-30_run1, 2 captures), each \u00d7 GEX+CITE+TCRab+TCRgd; the scRNA-seq analysis uses only the re-run (the first run gave 13,443 cells, the re-run 18,089); Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-05-20", + "display_name": "Longitudinal Blood 2026-05-20 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-05-20", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-06-16", + "display_name": "Longitudinal Blood 2026-06-16 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-06-16", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-06-22", + "display_name": "Longitudinal Blood 2026-06-22 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-06-22", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + }, + { + "sample_id": "blood_2026-07-01", + "display_name": "Longitudinal Blood 2026-07-01 (PBMC)", + "tissue": "Blood", + "timepoint": "", + "collection_date": "2026-07-01", + "collection_site": "", + "tissue_source": "PBMC from whole blood", + "vendors_involved": "", + "assays_run": "", + "bam_count": "0", + "bam_display_names": "", + "is_primary": "", + "is_superseded_by": "", + "notes": "Hudson Lab scRNA-seq, 3 captures \u00d7 GEX+CITE+TCRab+TCRgd; Draw site not documented; shipping format not documented (the Hudson report covers draws through 2025-11, which were shipped as whole blood from June 2025); collection_date is the Hudson folder date, taken as the draw date (inferred, owner decision 2026-10-02; awaiting Hudson Lab confirmation, beads osteosarc_com-azw.15); Cell Ranger BAMs exist under hudson_lab/PBMC_scRNAseq/cellranger/ (not catalogued)" + } + ], + "data_rows": [ + { + "context": "Bulk DNA Sequencing / Variant Calling (WGS)", + "row": { + "Timepoint": "Dec 2022 (T0)", + "Pipeline": "oncoanalyser 2.3.0 (2026-09 re-run)", + "Callers": "AMBER, COBALT, ESVEE, LILAC, LINX, PAVE, PURPLE, SAGE; RNA (BostonGene BG003082): Isofox, NEO, CUPPA", + "Bucket Path": "kamil/oncoanalyser/2026-09/IPISRC044_T0_personalis", + "Size": "13.3 GB", + "Files": "500" + } + }, + { + "context": "Bulk DNA Sequencing / Whole Exome Sequencing (WES) \u2014 FASTQ", + "row": { + "Timepoint": "Dec 2022 (T0)", + "Provider": "Tempus (xT 648-gene panel, not WES; GRCh37)", + "Tissue": "Tumor", + "Bucket Path": "rna-seq/tempus/TL-24-5GQLV9WSXQ/wes", + "Size": "12.6 GB", + "Files": "7" + } + }, + { + "context": "Bulk DNA Sequencing / Whole Exome Sequencing (WES) \u2014 BAM", + "row": { + "Timepoint": "Dec 2022 (T0)", + "Provider": "Boston Gene (T0 normal re-delivered with BG009368; 2024 Sarek run)", + "Tissue": "Normal (Blood)", + "Bucket Path": "genomics/genomics-duplicates/sarek24_bostongene/alignment/preprocessing/recalibrated/normal_bg24/normal_bg24.recal.bam", + "Size": "11.9 GB", + "Files": "1" + } + }, + { + "context": "Bulk DNA Sequencing / Whole Exome Sequencing (WES) \u2014 BAM", + "row": { + "Timepoint": "Dec 2022 (T0)", + "Provider": "Tempus (xT 648-gene panel, not WES; GRCh37)", + "Tissue": "Tumor", + "Bucket Path": "rna-seq/tempus/TL-24-5GQLV9WSXQ/wes/TL-24-5GQLV9WSXQ_T.sorted.bam", + "Size": "4.7 GB", + "Files": "1" + } + } + ] +} diff --git a/tests/test_alignment_inventory_time.py b/tests/test_alignment_inventory_time.py index 05f4b55..bcb23a9 100644 --- a/tests/test_alignment_inventory_time.py +++ b/tests/test_alignment_inventory_time.py @@ -6,7 +6,15 @@ import pytest -from osteosarc import Cache, File, IntegrityError, RecoveryPolicy, Region, extract_reads +from osteosarc import ( + Cache, + File, + IntegrityError, + RecoveryPolicy, + Region, + count_reads, + extract_reads, +) from osteosarc.cache import _check_inventory_time, stable_id, write_json from osteosarc.read_receipts import write_read_receipt from osteosarc.reads import inspect_alignment @@ -59,13 +67,15 @@ def acquire(operation, source, bam, cache): if operation == "inspect": return inspect_alignment(source, cache=cache, snapshot_id="frozen") kwargs = dict(cache=cache, index=str(bam) + ".bai", snapshot_id="frozen", max_records=6) + if operation == "count": + return count_reads(source, REGIONS, **kwargs) if operation == "recover": kwargs.pop("max_records") kwargs["recovery"] = RecoveryPolicy(max_rounds=1, max_records=100) return extract_reads(source, REGIONS, **kwargs) -@pytest.mark.parametrize("operation", ["inspect", "extract", "recover"]) +@pytest.mark.parametrize("operation", ["inspect", "extract", "recover", "count"]) @pytest.mark.parametrize("offset", [-86400, 86400]) def test_remote_inventory_date_mismatch_fails_before_reading(remote_bam, tmp_path, operation, offset): source, bam, state = remote_bam @@ -117,16 +127,17 @@ def test_invalid_server_date_is_an_integrity_error(remote_bam, tmp_path): assert not list(cache.workspace.rglob("receipt.json")) -@pytest.mark.parametrize("operation", ["inspect", "extract"]) +@pytest.mark.parametrize("operation", ["inspect", "extract", "count", "legacy_count"]) @pytest.mark.parametrize("legacy", [False, True]) def test_offline_cached_date_mismatch_is_rejected(remote_bam, tmp_path, monkeypatch, operation, legacy): import osteosarc.reads as reads source, bam, state = remote_bam cache = Cache(tmp_path / "cache") - result = acquire(operation, source, bam, cache) + result = acquire("extract" if operation == "legacy_count" else operation, source, bam, cache) receipt = result.receipt - directory = result.path.parent + directory = (cache.workspace / "derived" / stable_id(receipt["request"]) + if operation == "count" else result.path.parent) # Model a result accepted by the old implementation under a mismatched inventory. receipt["request"]["source_modified"] += 86400 if legacy: @@ -139,7 +150,21 @@ def test_offline_cached_date_mismatch_is_rejected(remote_bam, tmp_path, monkeypa monkeypatch.setattr(reads, "_remote_identity", lambda *a, **k: pytest.fail("offline HTTP access")) monkeypatch.setattr(reads, "_run", lambda *a, **k: pytest.fail("offline SAMtools execution")) with pytest.raises(IntegrityError, match="inventory lists.*new snapshot"): - acquire(operation, replace(source, modified=EPOCH + 86400), bam, Cache(cache.root, offline=True)) + acquire("count" if operation == "legacy_count" else operation, + replace(source, modified=EPOCH + 86400), bam, Cache(cache.root, offline=True)) + + +def test_matching_inventory_count_is_reusable_offline_without_bams(remote_bam, tmp_path, monkeypatch): + import osteosarc.reads as reads + + source, bam, state = remote_bam + cache = Cache(tmp_path / "cache") + count = acquire("count", source, bam, cache) + assert count.records == 6 + assert count.receipt["inventory_modification"] == "matched" + assert not list((cache.workspace / "derived").rglob("*.bam*")) + monkeypatch.setattr(reads, "_remote_identity", lambda *a, **k: pytest.fail("offline HTTP access")) + assert acquire("count", source, bam, Cache(cache.root, offline=True)) == count def test_size_check_still_precedes_reading(remote_bam, tmp_path): diff --git a/tests/test_bounded_pipe.py b/tests/test_bounded_pipe.py index fa8d0af..84f07c3 100644 --- a/tests/test_bounded_pipe.py +++ b/tests/test_bounded_pipe.py @@ -35,39 +35,50 @@ def producer(path, script): return [sys.executable, "-c", script, str(path), "-o", "-"] -def test_mid_header_timeout_has_no_seek_cleanup_error(large_header_bam, tmp_path, monkeypatch): +@pytest.mark.parametrize("count_only", [False, True]) +def test_mid_header_timeout_has_no_seek_cleanup_error(large_header_bam, tmp_path, monkeypatch, count_only): unraisable = [] monkeypatch.setattr(sys, "unraisablehook", lambda error: unraisable.append(str(error.exc_value))) script = ("import pathlib,sys,time; " "sys.stdout.buffer.write(pathlib.Path(sys.argv[1]).read_bytes()[:17400]); " "sys.stdout.buffer.flush(); time.sleep(10)") - output = tmp_path / "out.bam" + output = None if count_only else tmp_path / "out.bam" with pytest.raises(subprocess.TimeoutExpired): _run_bounded(producer(large_header_bam, script), output, 10, 0.5) assert not unraisable # A failed stream must not interfere with the next acquisition. - _run_bounded(["samtools", "view", "--no-PG", "-b", "-o", "-", str(large_header_bam)], - output, 3, 10) - assert record_multiset(output) == record_multiset(large_header_bam) + result = _run_bounded(["samtools", "view", "--no-PG", "-b", "-o", "-", str(large_header_bam)], + output, 3, 10) + if count_only: + assert result[1] == 3 + assert not (tmp_path / "out.bam").exists() + else: + assert record_multiset(output) == record_multiset(large_header_bam) -@pytest.mark.parametrize("limit", [1, 2, 3]) -def test_record_caps_preserve_stored_values(large_header_bam, tmp_path, monkeypatch, limit): +@pytest.mark.parametrize("count_only", [False, True]) +@pytest.mark.parametrize("limit", [None, 1, 2, 3]) +def test_record_caps_preserve_stored_values(large_header_bam, tmp_path, monkeypatch, limit, count_only): unraisable = [] monkeypatch.setattr(sys, "unraisablehook", lambda error: unraisable.append(str(error.exc_value))) command = ["samtools", "view", "--no-PG", "-b", "-o", "-", str(large_header_bam)] - output = tmp_path / "out.bam" - if limit < 3: + output = None if count_only else tmp_path / "out.bam" + if limit is not None and limit < 3: with pytest.raises(RecordLimitError): _run_bounded(command, output, limit, 10) else: - _run_bounded(command, output, limit, 10) - assert record_multiset(output) == record_multiset(large_header_bam) + result = _run_bounded(command, output, limit, 10) + if count_only: + assert result[1] == 3 + assert not (tmp_path / "out.bam").exists() + else: + assert record_multiset(output) == record_multiset(large_header_bam) assert not unraisable +@pytest.mark.parametrize("count_only", [False, True]) @pytest.mark.parametrize("compressed", [False, True]) -def test_truncated_blocks_are_integrity_errors(large_header_bam, tmp_path, compressed): +def test_truncated_blocks_are_integrity_errors(large_header_bam, tmp_path, compressed, count_only): damaged = tmp_path / "damaged.bam" if compressed: damaged.write_bytes(large_header_bam.read_bytes()[:17400]) @@ -76,14 +87,15 @@ def test_truncated_blocks_are_integrity_errors(large_header_bam, tmp_path, compr damaged.write_bytes(gzip.compress(raw[:-1])) script = "import pathlib,sys; sys.stdout.buffer.write(pathlib.Path(sys.argv[1]).read_bytes())" with pytest.raises(IntegrityError, match="Truncated"): - _run_bounded(producer(damaged, script), tmp_path / "out.bam", 10, 10) + _run_bounded(producer(damaged, script), None if count_only else tmp_path / "out.bam", 10, 10) -def test_producer_failure_keeps_stderr_after_partial_header(large_header_bam, tmp_path): +@pytest.mark.parametrize("count_only", [False, True]) +def test_producer_failure_keeps_stderr_after_partial_header(large_header_bam, tmp_path, count_only): script = ("import pathlib,sys; " "sys.stdout.buffer.write(pathlib.Path(sys.argv[1]).read_bytes()[:17400]); " "sys.stdout.buffer.flush(); sys.stderr.write('remote source failed'); sys.exit(7)") with pytest.raises(subprocess.CalledProcessError) as caught: - _run_bounded(producer(large_header_bam, script), tmp_path / "out.bam", 10, 10) + _run_bounded(producer(large_header_bam, script), None if count_only else tmp_path / "out.bam", 10, 10) assert caught.value.returncode == 7 assert caught.value.stderr == b"remote source failed" diff --git a/tests/test_curation_live_review.py b/tests/test_curation_live_review.py new file mode 100644 index 0000000..0d29ac5 --- /dev/null +++ b/tests/test_curation_live_review.py @@ -0,0 +1,128 @@ +"""Verified October source edits retain old layouts and reject later drift.""" + +import copy +import json +import warnings +from importlib.resources import files +from pathlib import Path + +import pytest + +from osteosarc import Table, parse_variants +from osteosarc.cache import digest +from osteosarc.curation import ASSAYS, CORRECTIONS, Curation, CurationWarning, normalize_tissue + +DATA = Path(__file__).parent / "data" +REVIEW = json.loads((DATA / "curation-review-2026-10-05.json").read_text()) +REVISIONS = json.loads(files("osteosarc").joinpath("data/curation-revisions.json").read_text()) +RULES = {c.id: c for c in CORRECTIONS if c.id in REVISIONS["corrections"]} +ATTACHED = {"ush2a-transposed-duplicate", "allele-FAM157A-p_W70_Q71ins_14", + "allele-COL3A1-Splice", "otud4-source-unavailable"} + + +def test_reviewed_layouts_have_no_stale_rules_or_source_mutations(): + raw = copy.deepcopy(REVIEW["sources"]) + before = copy.deepcopy(raw) + curated = Curation(RULES.values(), raw.get) + with warnings.catch_warnings(): + warnings.simplefilter("error", CurationWarning) + report = {r["id"]: r["status"] for r in curated.report()} + for source in raw: + curated.records(source) + assert report == {cid: "applied" if cid in ATTACHED else "fixed_upstream" for cid in RULES} + assert raw == before + assert curated.records("vafs")[0] == before["vafs"] + source = {r["id"]: r for r in curated.records("source_variants")[0]} + assert source["MAP2-chr2-209694768"]["variant_type"] == "delins" + assert source["DCHS2-chr4-154323273"]["variant_type"] == "complex" + assert source["GAPVD1-chr9-125301980"]["variant_type"] == "complex" + assert source["USH2A-chr1-215650752"]["upstream_merge"]["retired_id"] == "USH2A-chr1-215560752" + + +@pytest.mark.parametrize("cid,source,key,value,field,changed", [ + ("gene-symbol-TRMO", "source_variants", "id", "TRMO-chr9-97910412", "ref", "G"), + ("allele-DCHS2-chr4-154322488", "vafs", "variant_id", "DCHS2-chr4-154323273", "alt_reads", "1"), + ("allele-GAPVD1-chr9-125299105", "source_variants", "id", "GAPVD1-chr9-125301980", "alt", "C"), + ("allele-MAP2-chr2-209694768", "source_variants", "id", "MAP2-chr2-209694768", "pos", 209694769), + ("fam157a-withdrawn-protein", "source_variants", "id", "FAM157A-p_W70_Q71ins_14", "note", "Protein verified"), + ("allele-FAM157A-p_W70_Q71ins_14", "source_variants", "id", "FAM157A-p_W70_Q71ins_14", "refseq_id", "NM_000001"), + ("allele-COL3A1-Splice", "vafs", "variant_id", "COL3A1-Splice", "pos", "189010890"), + ("otud4-source-unavailable", "source_variants", "id", "OTUD4-p_A153del", "ref", "G"), + ("ush2a-transposed-duplicate", "source_variants", "id", "USH2A-chr1-215650752", "genomic_ref_context", "AAAA"), + ("specimen-T1-site", "specimens", "sample_id", "T1_tumor", "tissue_source", "Tumor resection"), + ("specimen-T3-site", "specimens", "sample_id", "T3_tumor", "collection_site", "UCSF"), + ("tempus-file-labels", "bam_metadata", "s3_path", "vendor/tempus/TL-24-ALMY2X4KMV/DNA/TL-24-ALMY2X4KMV_N.sorted.bam", "timepoint", "T0"), + ("reyagel-end-date", "events", "title", "ReyaGel", "end_date", "2027-07-14"), +]) +def test_unreviewed_future_changes_are_stale_and_atomic(cid, source, key, value, field, changed): + raw = copy.deepcopy(REVIEW["sources"]) + next(r for r in raw[source] if r.get(key) == value)[field] = changed + before = copy.deepcopy(raw) + curated = Curation([RULES[cid]], raw.get) + with pytest.warns(CurationWarning): + assert curated.report()[0]["status"] == "stale" + for name in raw: + assert curated.records(name)[0] == before[name] + + +@pytest.mark.parametrize("cid,source,key,restored", [ + ("map2-split-representations", "variant_index", "id", "MAP2-chr2-209694772"), + ("gene-symbol-TRMO", "source_variants", "id", "TMRO-chr9-97910412"), + ("tempus-grch37-counts", "vafs", "bam_file", "TL-24-5GQLV9WSXQ_T.sorted.bam"), +]) +def test_reintroduced_retired_records_do_not_match_a_reviewed_removal(cid, source, key, restored): + raw = copy.deepcopy(REVIEW["sources"]) + raw[source].append({key: restored}) + curated = Curation([RULES[cid]], raw.get) + with pytest.warns(CurationWarning): + assert curated.report()[0]["status"] == "stale" + + +def test_public_otud4_allele_has_primary_call_and_reference_evidence_without_somatic_claim(): + folder = DATA / "otud4-review" + provenance = json.loads((folder / "provenance.json").read_text()) + for filename, entry in provenance.items(): + assert digest(folder / filename) == entry["fixture_sha256"] + if entry["selection"].startswith("whole"): + assert entry["sha256"] == entry["fixture_sha256"] + raw = copy.deepcopy(REVIEW["sources"]) + curated = Curation(RULES.values(), raw.get) + variants = parse_variants(curated.records("variant_index")[0], Table(curated.records("vafs")[0]), + source_variants=curated.records("source_variants")[0]) + variant = variants["OTUD4-p_A153del"] + assert variant.status == "ready" and variant.allele == ("chr4", 145155970, "TCAG", "T") + evidence = variant.annotations["allele_resolution"] + assert evidence["source"]["sha256"] == provenance["cegat-normal-indels.vcf"]["sha256"] + assert evidence["annotation_source"]["sha256"] == provenance["cegat-normal-indels.annotated.tsv"]["sha256"] + row, = (folder / "cegat-normal-indels.vcf").read_text().splitlines() + fields = row.split("\t") + assert fields[:5] == ["chr4", "146077122", ".", "TCAG", "T"] + assert "DP=323" in fields[7] and "AD=7" in fields[7] + annotation, = [line.split("\t") for line in (folder / "cegat-normal-indels.annotated.tsv").read_text().splitlines() + if "NM_001102653.1" in line] + assert annotation[4] == evidence["source_filter"] == "FP/HET" + assert annotation[12:14] == [evidence["source_hgvs_c"], evidence["source_hgvs_p"]] + sequences = [] + for reference, allele in zip(evidence["references"], [evidence["source_allele"], evidence["allele"]]): + sequence = "".join((folder / (reference["accession"] + ".fasta")).read_text().splitlines()[1:]) + offset = allele["pos"] - reference["start"] + assert sequence[offset:offset + len(allele["ref"])] == allele["ref"] + sequences.append(sequence) + assert sequences[0] == sequences[1] + mapping, = json.loads((folder / "mapping.json").read_text())["mappings"] + assert mapping == evidence["mapping"]["mappings"][0] + assert mapping["mapped"]["start"] == evidence["allele"]["pos"] + assert evidence["original_natera_report"] == "unavailable" + assert evidence["somatic_status"] == "not_established" + label = json.loads((folder / "source-label.json").read_text()) + assert label["row"]["tissue"] == "Blood" and "normal DNA" in label["row"]["notes"] + assert evidence["source_tissue"] == "blood" + assert evidence["source_metadata"]["sha256"] == label["receipt"]["sha256"] + + +def test_reviewed_labels_preserve_assay_and_composite_tissue_distinctions(): + assert ASSAYS["Panel"] == ("panel", None) + assert normalize_tissue("blood normal (drawn 2024-06-11)") == "blood" + assert normalize_tissue("organoid (read 1)") == "organoid" + assert normalize_tissue("tumor + blood (drawn 2024-09-21)") is None + assert normalize_tissue("normal (blood, drawn 2027-01-01)") == "normal (blood, drawn 2027-01-01)" diff --git a/tests/test_reads.py b/tests/test_reads.py index 158b6d5..d31aefb 100644 --- a/tests/test_reads.py +++ b/tests/test_reads.py @@ -87,6 +87,173 @@ def test_cached_header_reuse_and_corruption(bam, tmp_path, monkeypatch): inspect_alignment(bam, cache=cache) +@pytest.mark.parametrize("limit", [None, 6]) +def test_regional_counts_preserve_multiplicity_without_storing_bams(bam, tmp_path, limit): + from osteosarc import count_reads + cache = Cache(tmp_path / "cache", offline=True) + regions = [Region("1", 100, 140, "GRCh38"), Region("chr1", 115, 160, "GRCh38")] + count = count_reads(bam, regions, cache=cache, max_records=limit) + assert count.records == 6 # overlapping regions count once; identical records remain twice + assert count.receipt["scope"] == "regional_record_count" + assert count.receipt["request"]["operation"] == "count_reads" + assert "count.json" in count.receipt["files"] + assert not list((cache.workspace / "derived").rglob("*.bam*")) + assert count_reads(bam, regions, cache=cache, filters=ReadFilter(exclude_flags=256 | 1024 | 2048)).records == 3 + assert count_reads(bam, [Region("chr2", 10, 20, "GRCh38")], cache=cache).records == 0 + with pytest.raises(CoordinateError): + count_reads(bam, [], cache=cache) + + +def test_count_cache_and_overflow_are_reusable_and_verified(bam, tmp_path, monkeypatch): + import json + + import osteosarc.reads as reads + from osteosarc import RecordLimitError, count_reads + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + count = count_reads(bam, regions, cache=cache, max_records=6) + with pytest.raises(RecordLimitError): + count_reads(bam, regions, cache=cache, max_records=5) + assert not list((cache.workspace / "derived").rglob("*.bam*")) + monkeypatch.setattr(reads, "_run_bounded", lambda *a: pytest.fail("repeated the count query")) + offline = Cache(cache.root, offline=True) + assert count_reads(bam, regions, cache=offline, max_records=6) == count + with pytest.raises(RecordLimitError): + count_reads(bam, regions, cache=offline, max_records=5) + receipt_path, = (cache.workspace / "derived").glob("*/receipt.json") + receipt = json.loads(receipt_path.read_text()) + receipt["records"] += 1 + receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(IntegrityError, match="count differs"): + count_reads(bam, regions, cache=offline, max_records=6) + + +def test_counts_reuse_older_extractions_and_overflow_results(bam, tmp_path, monkeypatch): + import osteosarc.reads as reads + from osteosarc import RecordLimitError, count_reads + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + subset = extract_reads(bam, regions, cache=cache, max_records=6) + with pytest.raises(RecordLimitError): + extract_reads(bam, regions, cache=cache, max_records=5) + monkeypatch.setattr(reads, "_run_bounded", lambda *a: pytest.fail("repeated the old probe")) + monkeypatch.setattr(reads, "_run", lambda *a: pytest.fail("started a process for cached reads")) + count = count_reads(bam, regions, cache=Cache(cache.root, offline=True), max_records=6) + assert count.records == 6 and count.receipt == subset.receipt + with pytest.raises(RecordLimitError): + count_reads(bam, regions, cache=cache, max_records=5) + assert len(list((cache.workspace / "derived").glob("*/receipt.json"))) == 1 + + +def test_dataset_counts_share_extraction_identity(bam, dataset): + from osteosarc import File + file = File("local", "source.bam", str(bam), "alignment", "bam") + regions = [Region("chr1", 100, 160, "GRCh38")] + subset = dataset.extract_reads(file, regions, max_records=6) + count = dataset.count_reads(file, regions, max_records=6) + assert count.records == 6 and count.receipt == subset.receipt + + +@pytest.mark.parametrize("records", [5, 7, -1, True, "6", None]) +def test_counts_reject_incorrect_legacy_record_metadata(bam, tmp_path, records): + import json + + from osteosarc import count_reads + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + subset = extract_reads(bam, regions, cache=cache, max_records=6) + receipt = dict(subset.receipt, records=records) + subset.receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(IntegrityError, match="BAM count differs"): + count_reads(bam, regions, cache=Cache(cache.root, offline=True), max_records=6) + + +@pytest.mark.parametrize("content", ['{"records": true}', '{"records": -1}', '{}', '[]', 'invalid']) +def test_count_cache_rejects_malformed_record_data(bam, tmp_path, content): + import json + + from osteosarc import count_reads + from osteosarc.cache import digest + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + count_reads(bam, regions, cache=cache) + receipt_path, = (cache.workspace / "derived").glob("*/receipt.json") + count_path = receipt_path.parent / "count.json" + count_path.write_text(content) + receipt = json.loads(receipt_path.read_text()) + receipt["files"]["count.json"] = digest(count_path) + receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(IntegrityError, match="cached regional count|count differs"): + count_reads(bam, regions, cache=Cache(cache.root, offline=True)) + + +@pytest.mark.parametrize("unpinned", ["reads.bam", "reads.bam.bai"]) +def test_counts_require_legacy_bam_and_index_integrity_pins(bam, tmp_path, unpinned): + import json + + from osteosarc import count_reads + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + subset = extract_reads(bam, regions, cache=cache, max_records=6) + receipt = subset.receipt + del receipt["files"][unpinned] + subset.receipt_path.write_text(json.dumps(receipt)) + with pytest.raises(IntegrityError, match="missing from its receipt"): + count_reads(bam, regions, cache=Cache(cache.root, offline=True), max_records=6) + + +@pytest.mark.parametrize("legacy", [False, True]) +def test_counts_reuse_compact_receipts_and_overflow_offline(bam, tmp_path, monkeypatch, legacy): + import osteosarc.reads as reads + from osteosarc import RecordLimitError, count_reads + from osteosarc.read_receipts import compact_read_cache + cache = Cache(tmp_path / "cache") + regions = [Region("chr1", 100, 160, "GRCh38")] + filters = ReadFilter(query_names=("repeated", "supplementary")) + acquire = extract_reads if legacy else count_reads + acquire(bam, regions, cache=cache, filters=filters, max_records=4) + with pytest.raises(RecordLimitError): + acquire(bam, regions, cache=cache, filters=filters, max_records=3) + compact_read_cache(cache) + monkeypatch.setattr(reads, "_run", lambda *a: pytest.fail("cached query started SAMtools")) + monkeypatch.setattr(reads, "_run_bounded", lambda *a: pytest.fail("cached query acquired records")) + offline = Cache(cache.root, offline=True) + count = count_reads(bam, regions, cache=offline, filters=filters, max_records=4) + assert count.records == 4 + with pytest.raises(RecordLimitError): + count_reads(bam, regions, cache=offline, filters=filters, max_records=3) + asset, = (cache.workspace / "query-names").glob("*.txt") + asset.write_text("changed\n") + with pytest.raises(IntegrityError, match="query-name asset"): + count_reads(bam, regions, cache=offline, filters=filters, max_records=4) + + +@pytest.mark.parametrize("limit", [5, 6]) +def test_count_queries_do_not_cache_results_from_changed_remote_sources(bam, tmp_path, monkeypatch, limit): + import osteosarc.reads as reads + from osteosarc import RecordLimitError, count_reads + url = "https://example.test/alignment.bam" + identity = {"etag": '"original"', "content-length": str(bam.stat().st_size), "last-modified": None} + run, bounded = reads._run, reads._run_bounded + monkeypatch.setattr(reads, "_remote_identity", lambda *a, **kw: dict(identity)) + monkeypatch.setattr(reads, "_run", lambda command, timeout: run( + [str(bam) if c == url else c for c in command], timeout)) + + def changed(command, output, maximum, timeout): + try: + return bounded([str(bam) if c == url else c for c in command], output, maximum, timeout) + finally: + identity["etag"] = '"replacement"' + + monkeypatch.setattr(reads, "_run_bounded", changed) + cache = Cache(tmp_path / "cache") + with pytest.raises(RecordLimitError if limit == 5 else IntegrityError): + count_reads(url, [Region("chr1", 100, 160, "GRCh38")], cache=cache, + index=str(bam) + ".bai", snapshot_id="snapshot", max_records=limit) + assert not list((cache.workspace / "derived").glob("*/receipt.json")) + assert not list((cache.workspace / "derived").glob("*.over-limit.json")) + + def test_remote_extraction_refuses_changed_header_source(bam, tmp_path, monkeypatch): import subprocess diff --git a/tests/test_sample_locations.py b/tests/test_sample_locations.py new file mode 100644 index 0000000..7a36998 --- /dev/null +++ b/tests/test_sample_locations.py @@ -0,0 +1,114 @@ +"""Current published sample locations, including filename prefixes and row aliases.""" + +import html +import json +from collections import Counter +from pathlib import Path + +from osteosarc import File, Files, Table +from osteosarc.catalog import build_files, fastq_prefix, matches_fastq, parse_data_paths +from osteosarc.curation import ASSAYS +from osteosarc.models import SampleClaim +from osteosarc.views import _fastq_folders, get_data_hints + +REVIEW = json.loads((Path(__file__).parent / "data/sample-locations-2026-10-07.json").read_text()) + + +def test_current_locations_link_every_fastq_without_mixing_tumor_and_normal(dataset, monkeypatch): + sources = [dict(row=row, corrections=(), bams=(), missing=(), fastqs=tuple( + r for r in REVIEW["fastqs"] if r["sample_id"] == row["sample_id"])) for row in REVIEW["specimens"]] + monkeypatch.setattr(dataset, "_sample_sources", sources) + files = build_files(dict(files=REVIEW["bucket"]), dict(categories=[]), Table([]), Table([]), + (("vendor/cegat/P116686_2_S000048", SampleClaim("data_page", assay="wes")),), + fastqs=REVIEW["fastqs"]) + dataset._tag_samples(files) + monkeypatch.setattr(dataset, "files", files) + assert len(dataset.samples) == 31 + for row in REVIEW["fastqs"]: + sample = dataset.samples[row["sample_id"]] + matched = [f for f in sample.files if matches_fastq(f.key, row["s3_folder"])] + assert len(matched) == int(row["file_count"]) == 2 + assert sum(f.size for f in matched) == int(row["total_size_bytes"]) + assert all(f.samples == (sample.id,) for f in matched) + assert all(f.resolved("assay") == ASSAYS[row["assay"]][0] for f in matched) + assert all(row in f.metadata["fastq_rows"] for f in matched) + shown = next(r for r in _fastq_folders(sample, sample.files) + if r["published_location"] == row["s3_folder"]) + assert shown["files"] == 2 and not shown["directory"] + assert shown["folder"] == fastq_prefix(row["s3_folder"]) + assert row["s3_folder"] in sample.fastq_folders # original alias stays visible + commands = get_data_hints(dataset, [], [shown]) + assert "--exclude '*'" in commands + for f in matched: + assert "--include " + Path(f.key).name in commands + assert not any(f.samples for f in files if f.kind != "reads") + samples = dataset.samples + assert Counter(s.tissue for s in samples) == {"blood": 24, "tumor": 5, "organoid": 2} + assert [(samples[s].date, samples[s].site) for s in + ("T0_tumor", "T1_tumor", "T2_tumor", "T3_tumor")] == [ + ("2022-12-16", "UCSF"), ("2024-06-06", "UCLA"), + ("2025-01-28", "UCLA"), ("2025-04-17", "MSKCC")] + assert samples["T2_organoid"].assays == ("wgs",) + assert "another person's" in samples["T2_organoid"].notes + assert "Composite of separate draws" in samples["T0_blood"].notes + assert samples["blood_2024-06-11"].date == "2024-06-11" + assert samples["blood_2024-09-21"].date == "2024-09-21" + + +def test_prefix_boundaries_sidecars_and_exact_files(): + prefix = "vendor/cegat/P116686_1" + assert matches_fastq(prefix + ".1.fastq.gz", prefix) + assert matches_fastq(prefix + "_R1.fq.gz", prefix) + assert matches_fastq(prefix + "/lane/read.fastq.gz", prefix) + assert matches_fastq(prefix + ".fastq.gz", prefix + ".fastq.gz") + assert not matches_fastq(prefix + "0.1.fastq.gz", prefix) + assert not matches_fastq(prefix + ".1.fastq.gz.seqkit-stats.tsv", prefix) + assert not matches_fastq(prefix + ".bam", prefix) + assert not matches_fastq("other/read.fastq.gz", prefix) + assert not matches_fastq("other/read.fastq.gz", "") + # Only the reviewed Tempus labels have aliases; no generic /T or /N guessing. + assert fastq_prefix("another/FastQ/T") == "another/FastQ/T" + + +def test_fastq_library_metadata_preserves_conflicting_exact_file_claims(): + key = "library/read.fastq.gz" + files = build_files(dict(files=[[key, 10, 0]]), dict(categories=[]), Table([]), Table([]), + ((key, SampleClaim("data_page", assay="wes")),), fastqs=[ + dict(s3_folder="library", assay="RNA", display_name="RNA library")]) + file = next(f for f in files if f.key == key) + assert file.conflicts["assay"] == ("rna-seq", "wes") + assert file.resolved("assay") is None + + +def test_directory_counts_exclude_indexes_and_keep_nested_fastqs(): + from osteosarc.models import Sample + + sample = Sample("s", fastq_folders=("reads/run",)) + files = Files([File(str(i), key, key, kind, fmt, size=size) for i, (key, kind, fmt, size) in enumerate([ + ("reads/run/lane/R1.fastq.gz", "reads", "fastq", 10), + ("reads/run/lane/R2.fastq.gz", "reads", "fastq", 20), + ("reads/run/report.tsv", "table", "tsv", 100), + ("reads/run2/R1.fastq.gz", "reads", "fastq", 200)])]) + row, = _fastq_folders(sample, files) + assert row["files"] == 2 and row["size"] == "30 B" + assert row["directory"] and row["folder"] == "reads/run/" + + +def test_data_page_notes_do_not_create_provider_or_assay_conflicts(): + claims = [] + for entry in REVIEW["data_rows"]: + row = entry["row"] + page = ("

" + html.escape(entry["context"]) + "

" + + "".join("" for k in row) + + "" + + "".join("" for v in row.values()) + + "
" + html.escape(k) + "
" + html.escape(v) + "
") + claims.extend(parse_data_paths(page)) + assert claims + for path, claim in claims: + if "normal_bg24" in path: + assert claim.provider == "BostonGene" and claim.assay == "wes" + elif "5GQLV9WSXQ" in path: + assert claim.provider == "Tempus" and claim.assay == "panel" + else: + assert "oncoanalyser" in path and claim.assay is None diff --git a/tests/test_shared.py b/tests/test_shared.py index 3309dcb..ec487d5 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -518,7 +518,7 @@ def test_kept_templates_keep_their_split_alignments_when_the_spec_asks(dataset, import shutil import osteosarc.shared as shared - from osteosarc import extract_reads, generate_bundle, inspect_alignment + from osteosarc import count_reads, extract_reads, generate_bundle, inspect_alignment from osteosarc.records import record_multiset from osteosarc.shared import build_shared_recipe, bundle_spec if shutil.which("samtools") is None: @@ -529,12 +529,18 @@ def test_kept_templates_keep_their_split_alignments_when_the_spec_asks(dataset, monkeypatch.setattr(dataset, "extract_reads", lambda file, regions, **kw: extract_reads( str(local), regions, cache=dataset.cache, fetch_pairs=kw.get("fetch_pairs", False), unplaced_mates=kw.get("unplaced_mates", True), max_records=kw.get("max_records"))) + monkeypatch.setattr(dataset, "count_reads", lambda file, regions, **kw: count_reads( + str(local), regions, cache=dataset.cache, max_records=kw.get("max_records"))) key = "rna-seq/reprocessed/BG003082/BG003082.Aligned.sortedByCoord.out.md.bam" spec = bundle_spec(dataset, "split", variants=["DYNC1H1-chr14-101980529"], files=[key]) plain = build_shared_recipe(spec, dataset, required={}, log=lambda text: None) spec["selection"]["supplementary"] = True logged = [] split = build_shared_recipe(spec, dataset, required={}, log=logged.append) + with monkeypatch.context() as context: + context.setattr(dataset, "count_reads", dataset.extract_reads) + legacy = build_shared_recipe(spec, dataset, required={}, log=lambda text: None) + assert split == legacy # counting-only changes storage, not the frozen recipe (source,) = split["sources"].values() assert split["selection"]["supplementary"] and "supplementary" not in plain["selection"] # The region reaches the alignment's first base, which is enough to read it, so a rebuild reads it too. diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 4b315af..ee3625b 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -116,7 +116,7 @@ def test_one_sample_shows_its_files_and_how_to_get_them(dataset): shell, python = sample_view(t1, dataset), sample_view(t1, dataset, python=True) assert shell.startswith("T1_tumor: ") and "Corrected by specimen-T1-site" in shell assert "BAMs, aligned reads (1)" in shell and t1.bams[0] in shell - assert "FASTQ folders, raw reads" in shell and t1.fastq_folders[0] + "/" in shell + assert "FASTQ locations, raw reads" in shell and t1.fastq_folders[0] + "/" in shell assert f"osteosarc download {t1.bams[0]} --to ." in shell and "aws s3 cp --recursive --no-sign-request" in shell assert f'data.download("{t1.bams[0]}", to=".")' in python and "osteosarc download" not in python assert repr(t1) == python @@ -216,7 +216,7 @@ def test_sample_overview_shows_filter_names_and_filters_by_assay(dataset, capsys "rna-seq, scrna-seq, cite-seq, New label" text = repr(dataset.samples.select(timepoint="T1", tissue="tumor")) assert "T1_tumor" in text and "T0_tumor" not in text and "T1_blood" not in text - assert "sequencing" in text and "FASTQ folders" in text + assert "sequencing" in text and "FASTQ locations" in text t1 = dataset.samples["T1_tumor"] assert sequencing_text(t1.sequencing) in repr(dataset.samples) assert repr(dataset.samples.select(timepoint="missing")) == "(no matching samples)"