ci(release): refuse a tag that does not match the version, and ship curated 1.0.0 notes

.github/workflows/release.yml

@@ -18,6 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      checks: read
     steps:
       # The tag comes from the dispatch form or the pushed ref, so it is
       # untrusted text: it reaches the shell through env only, and it must
@@ -43,45 +44,36 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Generate release notes
-        id: notes
+      # A release is named after the version the pipeline reports, so the tag
+      # must equal manifest.version in nextflow.config at the tagged commit,
+      # and a major or minor release needs its curated notes,
+      # docs/releases/<tag>.md, which become the release body.
+      - name: The tag matches manifest.version, and the notes exist
         env:
           TAG: ${{ steps.tag.outputs.tag }}
+        run: scripts/ci/check-release.sh tag "$TAG" --notes-out "${RUNNER_TEMP}/notes.md"
+
+      # Only a commit that main's CI passed is released: it must be on main,
+      # and every check run on it (this job left out) must have finished green.
+      - name: The tagged commit is on main and its checks passed
+        env:
+          TAG: ${{ steps.tag.outputs.tag }}
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
         run: |
-          # Verify the tag exists in the repo
-          if ! git rev-parse "${TAG}" >/dev/null 2>&1; then
-            echo "ERROR: Tag ${TAG} does not exist in this repository" >&2
+          sha=$(git rev-parse "${TAG}^{commit}")
+          if ! git merge-base --is-ancestor "$sha" origin/main; then
+            echo "ERROR: ${TAG} (${sha}) is not a commit on main" >&2
             exit 1
           fi
-
-          # Find previous tag
-          PREV_TAG=$(git tag --sort=-version:refname | grep '^v' | grep -v "^${TAG}$" | head -1)
-
-          if [ -n "$PREV_TAG" ]; then
-            RANGE="${PREV_TAG}..${TAG}"
-          else
-            RANGE="${TAG}"
-          fi
-
-          {
-            echo "notes<<EOF"
-            echo "## What's Changed"
-            echo ""
-            git log "$RANGE" --pretty=format:"- %s" --no-merges 2>/dev/null || echo "- Initial release"
-            echo ""
-            echo ""
-            echo "## Docker Images"
-            echo ""
-            echo "All image versions are listed in [\`versions.env\`](versions.env)."
-            echo ""
-            echo "To set up: \`./scripts/setup.sh /path/to/data\`"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
+          gh api --paginate "repos/${REPO}/commits/${sha}/check-runs?per_page=100" \
+            --jq '.check_runs[] | [.name, .status, (.conclusion // "")] | @tsv' > "${RUNNER_TEMP}/runs.tsv"
+          scripts/ci/check-release.sh checks "${RUNNER_TEMP}/runs.tsv" --ignore "$GITHUB_JOB"
 
       - name: Create GitHub Release
         uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
         with:
           tag_name: ${{ steps.tag.outputs.tag }}
           name: ${{ steps.tag.outputs.tag }}
-          body: ${{ steps.notes.outputs.notes }}
+          body_path: ${{ runner.temp }}/notes.md
           draft: false

CHANGELOG.md

@@ -0,0 +1,6 @@
+# Changelog
+
+Each release's notes. From 1.0.0 on, a major or minor release has hand-written notes under `docs/releases/`, and the release workflow uses them as the release body. Older releases list their commits on GitHub.
+
+- 1.0.0: [docs/releases/v1.0.0.md](docs/releases/v1.0.0.md)
+- 0.2.0 to 0.8.2: https://github.com/GeiserX/OwnGenome/releases

bin/render_report.py

@@ -1445,7 +1445,8 @@ def head(key, title=None):
     m = s["manifest"]
     if m:
         commit = next((r[2] for r in m if r[0] == "run" and r[1] == "git_commit"), "unknown")
-        w(f"Run manifest: run_manifest.tsv (pipeline commit {commit}, "
+        release = next((r[2] for r in m if r[0] == "run" and r[1] == "pipeline_version"), None)
+        w(f"Run manifest: run_manifest.tsv ({f'OwnGenome {release}, ' if release else ''}pipeline commit {commit}, "
           f"{sum(1 for r in m if r[0] == 'image')} images)")
     w("=" * 80)
     w("  DISCLAIMER: This is NOT a clinical report. Discuss findings with a")
@@ -3134,6 +3135,8 @@ def html_page(s, parts, index=None):
     S = s["sections"]
     m = s["manifest"]
     commit = next((r[2] for r in m if r[0] == "run" and r[1] == "git_commit"), None) if m else None
+    # The OwnGenome release the run declared (nextflow.config's manifest.version), when the manifest records it.
+    release = next((r[2] for r in m if r[0] == "run" and r[1] == "pipeline_version"), None) if m else None
     db = s["databases"]
     ns = s.get("new_since") or {}
     changed = set(ns.get("sections") or [])
@@ -3180,7 +3183,8 @@ def html_page(s, parts, index=None):
            f"    Sample: <strong>{E(s['sample'])}</strong> &nbsp;|&nbsp; Generated: {E(s['generated_utc'])}"
            + (f" &nbsp;|&nbsp; Latest run: {E(s['run']['started_utc'])}" if s["run"]["started_utc"] else ""),
            "  </div>", '  <div class="meta">',
-           f"    Pipeline commit: {E(commit or 'unknown')} &nbsp;|&nbsp; ClinVar file: {E(db['clinvar_release'] or 'unknown')}"
+           (f"    OwnGenome release: {E(release)} &nbsp;|&nbsp;" if release else "")
+           + f"    Pipeline commit: {E(commit or 'unknown')} &nbsp;|&nbsp; ClinVar file: {E(db['clinvar_release'] or 'unknown')}"
            f" &nbsp;|&nbsp; ClinVar in CPSR's bundle: {E(db['cpsr_clinvar_release'] or 'unknown')}"
            f" &nbsp;|&nbsp; HLA database: {E(db['hla_database'] or 'unknown')}",
            "  </div>"] + ([f'  <div class="meta">{E(recheck_line(s["recheck"]))}</div>'] if s.get("recheck") else []) + [

docs/releases/v1.0.0.md

@@ -0,0 +1,117 @@
+# OwnGenome 1.0.0
+
+> **Not released yet.** 1.0.0 is not tagged. Some entries below describe work still landing on main, and the compare link at the end works once the tag exists. The release removes this notice: the release checks refuse notes that still carry it.
+
+OwnGenome 1.0.0 is the first stable release. It is one Nextflow pipeline that takes your FASTQ, BAM or VCF and runs every step on your own machine. These notes cover everything since 0.8.2, grouped by what you get.
+
+## Before you upgrade
+
+- **Realign your samples.** The default reference is now NCBI's GRCh38 analysis set without ALT contigs. With ALT contigs, reads at CYP2D6, the MHC and KIR got mapping quality 0, so depth there was wrong. `validate-setup.sh` refuses a BAM aligned to another reference. How to move: https://geiserx.github.io/OwnGenome/realignment/
+- **Java 17+ and Nextflow are needed.** `run-all.sh` keeps its arguments, writes a one-row samplesheet and starts `nextflow run main.nf -resume`. The bash scheduler is gone. The numbered scripts still run one step each.
+- **The project has a new name: OwnGenome.** The repository, the docs site and the report carry it, with shims for the old command names.
+- **1.0 makes a promise.** Command options, config keys and output file names and formats stay as they are until a 2.0. A new column or file can be added in a 1.x; a rename or removal waits for 2.0. The output contract lists what is covered, and CI fails when a stable output breaks.
+
+## One pipeline from raw reads
+
+- The Nextflow pipeline starts from FASTQ: fastp, minimap2 alignment with duplicate marking and the BAM index in one pass, and DeepVariant writing a VCF and a gVCF. Before, Nextflow started from a VCF and the alignment existed only as bash.
+- The gVCF means PRS and PharmCAT see the sites where you match the reference, instead of treating them as missing.
+- The alignment fails when the BAM lost input reads, instead of passing a BAM with a third missing.
+- A VCF from a vendor is checked before any step runs: a GRCh37 file, a multi-sample file and a file without contig lengths are refused with the reason.
+- `run-all.sh` resumes, keeps per-step logs, starts from the BAM and VCF an earlier run produced, and runs the sample identity and contamination checks in a default run.
+- `bin/owngenome_recheck.py` says which results a newer ClinVar or catalog would change, and `--rerun` reruns only those.
+
+## Watching a run
+
+- A live site shows a run as it happens, from any device, behind a password built into the server. The invented demo sample lets anyone try it.
+- The run says which steps are planned, running and done, how long each step usually takes on this machine, and how many hours are left, as a range.
+- It warns before a task runs out of memory or a disk fills, and before a run stalls, on Linux, macOS and WSL2.
+
+## The report
+
+- The HTML report reads in seven chapters with a table of contents. One section prints on its own, with its caveats.
+- A confidence mark on each finding, and charts that keep low-star and low-confidence data visible.
+- One search box finds any gene, rsID or variant across every result, including repeat expansions, Locityper loci and the CFTR and UGT1A1 readings.
+- The report says what changed since the previous run of the same sample, and the run manifest records which databases and images produced it.
+- The report says which genes this genome could not read reliably, instead of showing them as normal, and says plainly what short reads cannot see.
+- Any finding opens in a local genome viewer with your reads, from the report or the live site. Nothing is uploaded.
+- A gene hit twice across engines is flagged, for example a deletion plus a variant on the other copy.
+- The Nextflow report gets every table the bash report reads: ClinVar frequencies and rsIDs, CPSR classes, read counts and the clinical and slivar detail.
+
+## Clinical findings
+
+- ClinVar hits need a depth and genotype-quality floor and a population frequency. Lab-construct contamination (cDNA reads clipped at exon edges) is flagged, and known deep intronic sites are always reported.
+- A delins the caller split into adjacent records matches the combined ClinVar record.
+- Heterozygous MUTYH or HFE carriers no longer show as secondary findings; deletions in secondary-finding genes do.
+- Exomiser ranks the variants that could explain a list of HPO terms.
+- The clinical filter ranks HIGH impacts on the MANE transcript and gives indels of unknown frequency their own tier.
+- Compound heterozygous pairs are phased from the reads: cis, trans or unknown.
+
+## Structural variants and copy number
+
+- A real SV merge replaces position binning. AnnotSV class 4 and 5 variants reach the reports with caller support, exon overlap and population frequency, and AnnotSV keeps genotype and phase so two deletions pair in trans.
+- Common repeat, immune-locus and MHC SVs, single-junction Manta breakends and long Delly calls that read depth contradicts no longer show as likely pathogenic.
+- A filtered or 0/0 Manta record no longer hides a deletion another caller found, and chrX depth is read by sex and PAR.
+- Copy number at any locus comes from one catalog row: HP, LCE3, CCL3L1, CYP21A2 deletions and gene conversions, and recurrent deletions such as 1q21.1 and AZF from a pinned ClinGen list. These results reach both reports and the couple screen.
+- Paraphase and Parascopy call alpha-globin, CYP21A2, GBA, PMS2, STRC, NCF1, OTOA and SMN1 copy number, and the report lists every paralog gene in the catalogs.
+- GRCh38's false duplications are masked, so genes such as U2AF1 can be read.
+- Mobile element insertions (Alu, L1, SVA), including the few known to cause disease.
+- Large mosaic gains, losses and copy-neutral LOH in blood, mosaic loss of chromosome Y or X, and large mtDNA deletions with the mtDNA copy count.
+
+## Repeats
+
+- 82 STRchive disease repeats are screened instead of the 31 ExpansionHunter bundles, with matching thresholds. Each locus shows its intermediate range and sizing confidence; RFC1 is resolved with a read-level motif count.
+- The CFTR 5T/TG tract and the UGT1A1 TA repeat are read directly from the reads.
+- The MUC1 repeat behind ADTKD is checked, which no standard caller sees.
+- Locityper genotypes the hardest polymorphic genes as an opt-in step.
+
+## Pharmacogenomics
+
+- pypgx calls deletions and hybrids for all 13 of its SV genes, shows copy-number calls with their confidence, and says when a gene cannot be called on this reference.
+- Drug guidance for HLA and CYP2D6 comes only when two callers agree. OptiType is an optional second HLA caller beside T1K.
+- An optional medication list reports CYP2D6 and CYP2C19 phenoconversion, and the report warns when a medicine changes a call or is contraindicated (for example valproate with POLG).
+- MT-RNR1 is called from the mtDNA variants. Indel-defined alleles are called reference when the gVCF holds a confident reference block over them.
+- HLA drug alleles and the DQ2.5, DQ2.2 and DQ8 types are derived from the T1K alleles.
+
+## Polygenic scores
+
+- Scores report an ancestry-adjusted percentile when the ancestry panel is installed, the ancestry each score was built on and how well it predicts.
+- Sex-specific scores are scored only for the matching sex, and chrX rows survive. Blood pressure and lipid scores are added.
+
+## Traits, blood groups and ancestry
+
+- A traits section: lactase persistence, caffeine, alcohol flush, taste, earwax, APOE and other replicated variants, each with its citation.
+- Multi-SNP results (APOE, APOL1, SERPINA1, HFE and others) come from one rule table checked against dbSNP, so a strand mistake fails CI. More risk-allele and marker rows are in the table.
+- Blood group antigens across ABO, Rh, Kell, Duffy, Kidd and the other systems, plus the platelet and neutrophil antigens that matter for transfusion and pregnancy.
+- Lp(a) from the two risk variants and the KIV-2 repeat count.
+- CHIP: clonal blood-cell mutations, with the detection limit at your depth.
+- Inherited integrated HHV-6 and viral DNA from the reads that do not map to the human genome.
+- Admixture fractions and chromosome painting against an open regional panel, placement among ancient populations, and your Neanderthal and Denisovan share and where it sits.
+- The Y line is placed on YFull's current tree, with the SNPs past the deepest known branch. Y-STR repeat counts, mtDNA and Y explore links, and the files those services accept.
+- Runs of homozygosity flag a chromosome that may come from one parent, and sex-chromosome aneuploidies are named.
+- A 23andMe-layout raw-data file from the gVCF, for third-party tools.
+- `bin/genome_lookup.py` answers "what is my genotype at rsX or in gene Y" without opening a VCF.
+
+## Families and couples
+
+- One family file says who is related to whom. The pipeline joint-calls the family, checks the declared relationships, phases a child by inheritance and maps which stretches each child got from each parent, ready for DNA Painter.
+- Shared DNA segments between two sequenced relatives, on the report and the live site.
+- An optional couple screen reports shared recessive and X-linked risk. Common population deletions, short Delly calls and megabase artefacts no longer count as carrier risk.
+- A daughter's chrX site that differs from her mother is not called de novo when her father carries the allele.
+- Chip data: mitochondrial sites lift over correctly, and MyHeritage files with Windows line endings convert.
+
+## Setup and reliability
+
+- `setup.sh` installs the VEP cache up front, retries a download whose checksum or gzip check fails, and keeps digest-pinned images through a cleanup.
+- Steps stop trusting half-written files, and a failed upstream command fails its task instead of leaving a short file that looks valid.
+- Memory budgets match a real genome for GLnexus, MEI and the PRS steps, and DeepVariant gets its full run time.
+- Every image version is one line in `versions.env`. Renovate proposes each update as its own pull request, and CI runs every changed image on a small real-data fixture.
+
+## Testing
+
+- The e2e suite runs the real tools on an HG002 slice, in parallel jobs, scoped to what a pull request can affect. A case that checks nothing fails.
+- A misspelt Nextflow parameter is caught instead of silently ignored.
+- A release tag must match the version the pipeline declares, and a commit main's CI did not pass cannot be released.
+
+Full commit list: https://github.com/GeiserX/OwnGenome/compare/v0.8.2...v1.0.0
+
+Get started: https://geiserx.github.io/OwnGenome/getting-started/

docs/research/nf-core-pipeline-requirements.md

@@ -18,7 +18,8 @@ The report below is a research snapshot. This table says which of its requiremen
 | Parameter schema validated at run time (nf-schema plugin) | unmet | no `plugins` block; `nextflow_schema.json` is draft-07 and only syntax-checked in CI |
 | `modules.config` for publishing, `meta.yml` per module, `modules.json` | unmet | every module hard-codes its `publishDir`; all modules are local |
 | nf-test tests | unmet | no `nf-test.config`; tests are the stub run, the e2e job and scripts under `tests/` |
-| `CHANGELOG.md` and `CITATIONS.md` | unmet | release notes live in GitHub releases |
+| `CHANGELOG.md` | met | `CHANGELOG.md` links each release's notes; from 1.0.0 they are hand-written in `docs/releases/` and `scripts/ci/check-release.sh` checks them |
+| `CITATIONS.md` | unmet | no `CITATIONS.md` |
 | Zero failures in `nf-core pipelines lint` | unmet | no lint job, and the repo is not built from the template |
 | Built from the nf-core template, `master`/`dev`/`TEMPLATE` branches | not planned | the pipeline predates the template; one `main` branch |
 | MIT license | not planned | GPL-3.0-or-later |