diff --git a/assets/output_contract.tsv b/assets/output_contract.tsv new file mode 100644 index 00000000..cca7450b --- /dev/null +++ b/assets/output_contract.tsv @@ -0,0 +1,425 @@ +# assets/output_contract.tsv: what OwnGenome 1.0 keeps stable (docs/output-contract.md). +# scripts/ci/check-output-contract.py reads it: --static in Guard's unit-tests job (tests/test_output_contract.py) +# checks every row against the code, and the e2e case output-contract-1-hg002 checks the files the +# HG002 run wrote. Columns, tab-separated: +# kind file, param (a nextflow_schema.json param), tool (a --tools name in main.nf) or +# switch (a run-all.sh switch, its SWITCHES list) +# name file: its path under the results folder (GENOME_DIR for the bash steps and run-all.sh, +# --outdir for nextflow run), {sample} for the sample id, {family} for --family_name; +# * where the name varies. Other kinds: the name as typed. +# step the step number (assets/steps.tsv), - for none +# process the Nextflow process that publishes the file, - when no module does (a bash step, +# run-all.sh or main.nf itself) +# legs file: who writes it under this name: nextflow (main.nf and run-all.sh), bash (the +# single step script), both. Other kinds: - +# format file: tsv, csv, vcf, json, text, html, bam, cram, index, bed, dir, zip, svg, png, md, +# yml, fasta, bin. Other kinds: - +# stability stable: kept until 2.0; experimental: an opt-in engine new in 1.0, listed but free to +# change in a 1.x release; internal: a task file that is not published, listed so the +# list stays complete +# e2e the leg (bash, nextflow, both) whose HG002 e2e run must write the file, - for none +# required tsv and csv: the columns; vcf: the INFO/ and FORMAT/ fields of its header; json: the +# top-level keys; comma-separated, - for none. More columns may come; none of these may go. +kind name step process legs format stability e2e required +file multiqc/multiqc_report.html 28 MULTIQC nextflow html stable - - +file multiqc/multiqc_report_data 28 MULTIQC nextflow dir stable - - +file pipeline_info/software_versions.yml - - nextflow yml stable nextflow - +file somalier/somalier.html 33 SOMALIER_RELATE nextflow html experimental - - +file somalier/somalier.pairs.tsv 33 SOMALIER_RELATE nextflow tsv experimental - - +file somalier/somalier.samples.tsv 33 SOMALIER_RELATE nextflow tsv experimental - - +file {family}/compound_het.tsv 38 FAMILY_REPORT nextflow tsv experimental - - +file {family}/couple_*.md 39 COUPLE_CARRIER nextflow md experimental - - +file {family}/couple_*.tsv 39 COUPLE_CARRIER nextflow tsv experimental - - +file {family}/de_novo.tsv 38 FAMILY_REPORT nextflow tsv experimental - - +file {family}/family.ped 38 FAMILY_PREPARE nextflow tsv experimental - - +file {family}/family.phased.vcf.gz 38 WHATSHAP_PED nextflow vcf experimental - - +file {family}/family.phased.vcf.gz.tbi 38 WHATSHAP_PED nextflow index experimental - - +file {family}/family.vcf.gz 38 GLNEXUS_VCF nextflow vcf experimental - - +file {family}/family.vcf.gz.tbi 38 GLNEXUS_VCF nextflow index experimental - - +file {family}/family_plan.tsv 38 FAMILY_PREPARE nextflow tsv experimental - - +file {family}/family_report.md 38 FAMILY_REPORT nextflow md experimental - - +file {family}/ibd_pairs.tsv 38 FAMILY_IBD_REPORT nextflow tsv experimental - - +file {family}/ibd_report.md 38 FAMILY_IBD_REPORT nextflow md experimental - - +file {family}/ibd_segments.tsv 38 FAMILY_IBD_REPORT nextflow tsv experimental - - +file {family}/members.tsv 38 FAMILY_PREPARE nextflow tsv experimental - - +file {family}/pedigree.tsv 38 FAMILY_PREPARE nextflow tsv experimental - - +file {family}/relationships.tsv 38 FAMILY_CHECK nextflow tsv experimental - - +file {family}/somalier/family.html 38 FAMILY_RELATE nextflow html experimental - - +file {family}/somalier/family.pairs.tsv 38 FAMILY_RELATE nextflow tsv experimental - - +file {family}/somalier/family.samples.tsv 38 FAMILY_RELATE nextflow tsv experimental - - +file {family}/trios.ped 38 FAMILY_PREPARE nextflow tsv experimental - - +file {family}/{sample}_dnapainter.csv 38 FAMILY_REPORT nextflow csv experimental - - +file {family}/{sample}_inheritance.tsv.gz 38 FAMILY_REPORT nextflow tsv experimental - - +file {family}/{sample}_segments.svg 38 FAMILY_REPORT nextflow svg experimental - - +file {family}/{sample}_segments.tsv 38 FAMILY_REPORT nextflow tsv experimental - - +file {sample}/aligned/{sample}_read_count.tsv 02 ALIGN_MINIMAP2 nextflow tsv stable - - +file {sample}/aligned/{sample}_sorted.bam 02 ALIGN_MINIMAP2 both bam stable bash - +file {sample}/aligned/{sample}_sorted.bam.bai 02 ALIGN_MINIMAP2 both index stable bash - +file {sample}/aligned/{sample}_sorted.cram 34 CRAM_ARCHIVE nextflow cram experimental - - +file {sample}/aligned/{sample}_sorted.cram.crai 34 CRAM_ARCHIVE nextflow index experimental - - +file {sample}/aligned/{sample}_sorted.cram.flagstat 34 CRAM_ARCHIVE nextflow bin experimental - - +file {sample}/ancestry/{sample}_ancestry.tsv 25 PRS_SUMMARY nextflow tsv stable - - +file {sample}/ancestry_fine/{sample}_ancestry_fine.tsv 26b ANCESTRY_FINE_REPORT nextflow tsv experimental - - +file {sample}/ancestry_fine/{sample}_ancestry_fine.txt 26b ANCESTRY_FINE_REPORT nextflow text experimental - - +file {sample}/ancestry_fine/{sample}_painting.bed 26b ANCESTRY_FINE_REPORT nextflow bed experimental - - +file {sample}/ancestry_fine/{sample}_painting.png 26b ANCESTRY_FINE_REPORT nextflow png experimental - - +file {sample}/ancient/{sample}_ancient.tsv 44 ANCIENT_REPORT nextflow tsv experimental - - +file {sample}/ancient/{sample}_ancient.txt 44 ANCIENT_REPORT nextflow text experimental - - +file {sample}/ancient/{sample}_ancient_info.tsv 44 ANCIENT_REPORT nextflow tsv experimental - - +file {sample}/ancient/{sample}_ancient_pca.svg 44 ANCIENT_REPORT nextflow svg experimental - - +file {sample}/annotsv/{sample}_annotsv_input.vcf 05 ANNOTSV nextflow vcf stable - - +file {sample}/annotsv/{sample}_sv_annotated.tsv 05 ANNOTSV nextflow tsv stable - - +file {sample}/annotsv/{sample}_sv_class4_5.tsv 05 ANNOTSV nextflow tsv stable - - +file {sample}/annotsv/{sample}_sv_class4_5_checked.tsv 05 ANNOTSV nextflow tsv stable - - +file {sample}/archaic/{sample}_archaic.tsv 43 ARCHAIC_REPORT nextflow tsv experimental - - +file {sample}/archaic/{sample}_archaic.txt 43 ARCHAIC_REPORT nextflow text experimental - - +file {sample}/archaic/{sample}_archaic_segments.tsv 43 ARCHAIC_REPORT nextflow tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype.tsv 41 BLOODTYPE nextflow tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype_depth.tsv 41 BLOODTYPE_DEPTH nextflow tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype_evidence.tsv 41 BLOODTYPE nextflow tsv experimental - - +file {sample}/chip/{sample}_chip.tsv 49 CHIP_REPORT nextflow tsv experimental - - +file {sample}/chip/{sample}_chip_depth.tsv 49 CHIP_REPORT nextflow tsv experimental - - +file {sample}/chip/{sample}_chip_filtered.vcf.gz 49 CHIP_MUTECT2 nextflow vcf experimental - - +file {sample}/clinical/{sample}_clinical.vcf.gz 23 CLINICAL_FILTER both vcf stable bash - +file {sample}/clinical/{sample}_clinical.vcf.gz.tbi 23 CLINICAL_FILTER nextflow index stable - - +file {sample}/clinical/{sample}_clinical_summary.tsv 23 CLINICAL_FILTER nextflow tsv stable - - +file {sample}/clinvar/{sample}_clinvar_cdna_check.tsv 06 CLINVAR_CDNA_CHECK nextflow tsv stable - - +file {sample}/clinvar/{sample}_clinvar_cdna_scan.tsv 06 CLINVAR_CDNA_CHECK nextflow tsv stable - - +file {sample}/clinvar/{sample}_clinvar_deep_intronic.tsv 06 CLINVAR_SCREEN nextflow tsv stable - - +file {sample}/clinvar/{sample}_clinvar_hits.tsv 06 CLINVAR_SCREEN both tsv stable both chrom,pos,ref,alt,genotype,clinvar_id,geneinfo,clnsig,clnrevstat +file {sample}/clinvar/{sample}_clinvar_hits.vcf 06 CLINVAR_SCREEN both vcf stable both INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT +file {sample}/clinvar/{sample}_pass.vcf.gz 06 CLINVAR_SCREEN both vcf stable both FORMAT/GT +file {sample}/clinvar/{sample}_pass.vcf.gz.tbi 06 CLINVAR_SCREEN both index stable both - +file {sample}/cnvpytor/{sample}_cnvs.raw.vcf 18 CNVPYTOR nextflow vcf internal - - +file {sample}/cnvpytor/{sample}_cnvs.txt 18 CNVPYTOR nextflow text stable - - +file {sample}/cnvpytor/{sample}_cnvs.vcf.gz 18 CNVPYTOR_VCF nextflow vcf stable - - +file {sample}/cnvpytor/{sample}_cnvs.vcf.gz.tbi 18 CNVPYTOR_VCF nextflow index stable - - +file {sample}/consumer_export/{sample}_23andme_v5.zip 40 CONSUMER_EXPORT nextflow zip experimental - - +file {sample}/consumer_export/{sample}_ancestrydna_layout.zip 40 CONSUMER_EXPORT nextflow zip experimental - - +file {sample}/consumer_export/{sample}_consumer_export.tsv 40 CONSUMER_EXPORT nextflow tsv experimental - - +file {sample}/couple/{sample}_couple_carrier.tsv 39 COUPLE_CARRIER nextflow tsv experimental - - +file {sample}/coverage/{sample}.mosdepth.global.dist.txt 16b MOSDEPTH nextflow text stable - - +file {sample}/coverage/{sample}.mosdepth.summary.txt 16b MOSDEPTH nextflow tsv stable nextflow chrom,length,bases,mean,min,max +file {sample}/coverage/{sample}.regions.bed.gz 16b MOSDEPTH nextflow bed stable - - +file {sample}/coverage/{sample}.thresholds.bed.gz 16b MOSDEPTH nextflow bed stable - - +file {sample}/coverage/{sample}_callability.bed.gz 16b MOSDEPTH nextflow bed stable - - +file {sample}/cpic/{sample}_cpic_recommendations.txt 27 CPIC_LOOKUP both text stable both - +file {sample}/cpic/{sample}_phenotypes.tsv 27 CPIC_LOOKUP both tsv stable both - +file {sample}/cpsr/{sample}.cpsr.grch38.classification.tsv.gz 17 CPSR nextflow tsv stable - - +file {sample}/cpsr/{sample}.cpsr.grch38.html 17 CPSR nextflow html stable - - +file {sample}/cyrius/{sample}_cyp2d6.tsv 21 CYRIUS nextflow tsv stable - - +file {sample}/cyrius/{sample}_cyp2d6_depth_check.tsv 21 CYRIUS nextflow tsv stable - - +file {sample}/delly/{sample}_sv.vcf.gz 19 DELLY_BCF2VCF nextflow vcf stable nextflow INFO/SVTYPE +file {sample}/delly/{sample}_sv.vcf.gz.tbi 19 DELLY_BCF2VCF nextflow index stable nextflow - +file {sample}/exomiser/{sample}_exomiser.txt 48 EXOMISER_REPORT nextflow text experimental - - +file {sample}/exomiser/{sample}_exomiser.variants.tsv 48 EXOMISER nextflow tsv experimental - - +file {sample}/exomiser/{sample}_exomiser_terms.tsv 48 EXOMISER_REPORT nextflow tsv experimental - - +file {sample}/exomiser/{sample}_exomiser_top.tsv 48 EXOMISER_REPORT nextflow tsv experimental - - +file {sample}/expansion_hunter/{sample}.json 09 EXPANSION_HUNTER nextflow json stable - - +file {sample}/expansion_hunter/{sample}.vcf 09 EXPANSION_HUNTER nextflow vcf stable - - +file {sample}/expansion_hunter/{sample}_eh_stranger.vcf 09b STRANGER nextflow vcf stable - - +file {sample}/expansion_hunter/{sample}_realigned.bam 09 EXPANSION_HUNTER nextflow bam stable - - +file {sample}/expansion_hunter/{sample}_rfc1_motif.tsv 09b STRANGER_RFC1 nextflow tsv stable - - +file {sample}/expansion_hunter/{sample}_str_motifs.tsv 09b STRANGER_RFC1 nextflow tsv stable - - +file {sample}/fastq_trimmed/{sample}_R1.fastq.gz 01b FASTP nextflow bin internal - - +file {sample}/fastq_trimmed/{sample}_R2.fastq.gz 01b FASTP nextflow bin internal - - +file {sample}/fastq_trimmed/{sample}_fastp.html 01b FASTP nextflow html stable - - +file {sample}/fastq_trimmed/{sample}_fastp.json 01b FASTP nextflow json stable - - +file {sample}/hla/{sample}_hla_genotype.tsv 08 HLA_TYPING nextflow tsv stable - - +file {sample}/hla_optitype/{sample}_hla_concordance.tsv 08 HLA_QC nextflow tsv experimental - - +file {sample}/hla_optitype/{sample}_optitype_result.tsv 08 HLA_QC nextflow tsv experimental - - +file {sample}/hla_t1k/{sample}_hla_genotype.tsv 08 - bash tsv stable bash - +file {sample}/indexcov 16 INDEXCOV nextflow dir stable - - +file {sample}/indexcov/indexcov-indexcov.ped 16 - bash tsv stable bash - +file {sample}/kir/database_release.txt 08 KIR_TYPING nextflow text experimental - - +file {sample}/kir/{sample}_kir_genotype.tsv 08 KIR_TYPING nextflow tsv experimental - - +file {sample}/logs/run_status.tsv - - nextflow tsv stable - - +file {sample}/lpa/{sample}_lpa_kiv2.tsv 46 LPA nextflow tsv experimental - - +file {sample}/lpa/{sample}_lpa_kiv2_depth.tsv 46 LPA_DEPTH nextflow tsv experimental - - +file {sample}/lpa/{sample}_lpa_kiv2_depth_q1.tsv 46 LPA_DEPTH nextflow tsv experimental - - +file {sample}/manta/results/variants/candidateSV.vcf.gz 04 MANTA nextflow vcf stable - - +file {sample}/manta/results/variants/diploidSV.raw.vcf.gz 04 MANTA nextflow vcf stable - - +file {sample}/manta/results/variants/diploidSV.raw.vcf.gz.tbi 04 MANTA nextflow index stable - - +file {sample}/manta/results/variants/diploidSV.vcf.gz 04 MANTA nextflow vcf stable - - +file {sample}/manta/results/variants/diploidSV.vcf.gz.tbi 04 MANTA nextflow index stable - - +file {sample}/mei/{sample}_megane.log 50 MEI nextflow text experimental - - +file {sample}/mei/{sample}_megane.vcf 50 MEI nextflow vcf experimental - - +file {sample}/mei/{sample}_mei.tsv 50 MEI_REPORT nextflow tsv experimental - - +file {sample}/mei/{sample}_mei_annotsv.tsv 50 MEI_REPORT nextflow tsv experimental - - +file {sample}/mei/{sample}_mei_depth.tsv 50 MEI_DEPTH nextflow tsv experimental - - +file {sample}/mei/{sample}_mei_mainchr.txt 50 MEI_DEPTH nextflow text internal - - +file {sample}/mei/{sample}_mei_sites.tsv 50 MEI_REPORT nextflow tsv experimental - - +file {sample}/mito/{sample}_chrM.fasta 12 MITO_EXTRACT_CHRM nextflow fasta stable - - +file {sample}/mito/{sample}_chrM.vcf.gz 12 MITO_EXTRACT_CHRM nextflow vcf internal - - +file {sample}/mito/{sample}_chrM.vcf.gz.tbi 12 MITO_EXTRACT_CHRM nextflow index internal - - +file {sample}/mito/{sample}_chrM_annotated.tsv 20 MITO_VARIANTS_ANNOTATE nextflow tsv stable - - +file {sample}/mito/{sample}_chrM_deletions.tsv 20 MITO_DELETIONS nextflow tsv stable - - +file {sample}/mito/{sample}_chrM_filtered.vcf.gz 20 MITO_VARIANTS both vcf stable bash - +file {sample}/mito/{sample}_chrM_filtered.vcf.gz.tbi 20 MITO_VARIANTS nextflow index stable - - +file {sample}/mito/{sample}_chrM_mutect2.vcf.gz.stats 20 MITO_VARIANTS nextflow bin stable - - +file {sample}/mito/{sample}_haplocheck.txt 12 HAPLOCHECK nextflow text stable - - +file {sample}/mito/{sample}_haplogroup.txt 12 MITO_HAPLOGROUP both text stable bash - +file {sample}/mito/{sample}_mtdna_copies.tsv 20 MITO_DELETIONS nextflow tsv stable - - +file {sample}/mosaic/{sample}_mocha.log 51 MOCHA_CALL nextflow text experimental - - +file {sample}/mosaic/{sample}_mocha.tsv 51 MOCHA_REPORT nextflow tsv experimental - - +file {sample}/mosaic/{sample}_mocha_calls_raw.tsv 51 MOCHA_CALL nextflow tsv experimental - - +file {sample}/mosaic/{sample}_mocha_sites.tsv 51 MOCHA_CALL nextflow tsv experimental - - +file {sample}/mosaic/{sample}_mocha_stats.tsv 51 MOCHA_CALL nextflow tsv experimental - - +file {sample}/mosaic/{sample}_mocha_summary.tsv 51 MOCHA_REPORT nextflow tsv experimental - - +file {sample}/locityper/{sample}_locityper.log 53 LOCITYPER both text experimental - - +file {sample}/locityper/{sample}_locityper.tsv 53 LOCITYPER both tsv experimental - - +file {sample}/locityper/work 53 LOCITYPER both dir experimental - - +file {sample}/mosdepth/{sample}.mosdepth.summary.txt 16b - bash tsv stable bash chrom,length,bases,mean,min,max +file {sample}/paralogs/{sample}_*_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/paralogs/{sample}_cn_dosage.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/paralogs/{sample}_cn_intervals.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/paralogs/{sample}_cyp21a2_psv.tsv 35 PARASCOPY both tsv experimental - - +file {sample}/paralogs/{sample}_hba_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/paralogs/{sample}_paraphase 35b PARAPHASE nextflow dir experimental - - +file {sample}/paralogs/{sample}_paraphase_copy_number.tsv 35b PARAPHASE_REPORT nextflow tsv experimental - - +file {sample}/paralogs/{sample}_paraphase_variants.tsv 35b PARAPHASE_REPORT nextflow tsv experimental - - +file {sample}/paralogs/{sample}_parascopy 35 PARASCOPY nextflow dir experimental - - +file {sample}/paralogs/{sample}_smn_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/pgx_consensus/{sample}_outside_calls.tsv 36 PGX_CONSENSUS nextflow tsv stable - - +file {sample}/pgx_consensus/{sample}_pgx_consensus.tsv 36 PGX_CONSENSUS nextflow tsv stable - - +file {sample}/pharmcat/{sample}.match.json 07 PHARMCAT nextflow json stable - - +file {sample}/pharmcat/{sample}.phenotype.json 07 PHARMCAT nextflow json stable - - +file {sample}/pharmcat/{sample}.report.html 07 PHARMCAT nextflow html stable - - +file {sample}/pharmcat/{sample}.report.json 07 PHARMCAT nextflow json stable nextflow - +file {sample}/prs/pgsc_calc 25 PRS nextflow dir stable - - +file {sample}/prs/{sample}_prs_summary.*.tsv 25 PRS_SUMMARY nextflow tsv stable - - +file {sample}/prs/{sample}_prs_summary.tsv 25 PRS_SUMMARY nextflow tsv stable - - +file {sample}/pypgx/{sample}_cyp2d6_depth_check.tsv 32 PYPGX nextflow tsv stable - - +file {sample}/pypgx/{sample}_pharmcat_comparison.tsv 27 CPIC_LOOKUP nextflow tsv stable - - +file {sample}/pypgx/{sample}_pypgx_results 32 PYPGX nextflow dir stable - - +file {sample}/pypgx/{sample}_pypgx_summary.tsv 32 PYPGX both tsv stable bash - +file {sample}/qc/somalier/{sample}.somalier 33 SOMALIER nextflow bin experimental - - +file {sample}/qc/somalier/{sample}.somalier_id 33 SOMALIER nextflow bin internal - - +file {sample}/qc/verifybamid2/{sample}.Ancestry 33 VERIFYBAMID2 nextflow bin experimental - - +file {sample}/qc/verifybamid2/{sample}.log 33 VERIFYBAMID2 nextflow text experimental - - +file {sample}/qc/verifybamid2/{sample}.marker_check 33 VERIFYBAMID2 nextflow bin internal - - +file {sample}/qc/verifybamid2/{sample}.selfSM 33 VERIFYBAMID2 nextflow bin experimental - - +file {sample}/qc/{sample}_sample_qc.tsv 33 SAMPLE_QC nextflow tsv experimental - - +file {sample}/roh/{sample}_roh.txt 11 ROH nextflow text stable nextflow - +file {sample}/roh/{sample}_roh_summary.txt 11 ROH nextflow tsv stable nextflow chrom,start,end,length_bp,length_mb +file {sample}/run_manifest.tsv - - nextflow tsv stable - - +file {sample}/slivar/{sample}.ped 31 SLIVAR_PRIORITIZE nextflow tsv internal - - +file {sample}/slivar/{sample}_compound_het_pairs.tsv 31 COMPHET_PHASE nextflow tsv stable - - +file {sample}/slivar/{sample}_compound_hets.vcf.gz 31 SLIVAR both vcf stable bash - +file {sample}/slivar/{sample}_prioritized.vcf.gz 31 SLIVAR_PRIORITIZE both vcf stable bash - +file {sample}/slivar/{sample}_prioritized.vcf.gz.tbi 31 SLIVAR_PRIORITIZE nextflow index stable - - +file {sample}/slivar/{sample}_slivar_summary.tsv 31 SLIVAR_PRIORITIZE nextflow tsv stable - - +file {sample}/summary.json 24 HTML_REPORT both json stable bash schema_version,sample,generated_utc,run,databases,manifest,sections,cyp2d6,not_run,not_assessed +file {sample}/sv_duphold/{sample}_sv_duphold.vcf 15 DUPHOLD nextflow vcf stable - - +file {sample}/sv_filtered/{sample}_sv_filtered.log 15 DUPHOLD_FILTER nextflow text stable - - +file {sample}/sv_filtered/{sample}_sv_filtered.vcf.gz 15 DUPHOLD_FILTER nextflow vcf stable - - +file {sample}/sv_filtered/{sample}_sv_filtered.vcf.gz.tbi 15 DUPHOLD_FILTER nextflow index stable - - +file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz 22 SURVIVOR_SORT nextflow vcf stable - - +file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz.tbi 22 SURVIVOR_SORT nextflow index stable - - +file {sample}/telomere/{sample} 10 TELOMERE_HUNTER nextflow dir stable - - +file {sample}/traits/{sample}_genotype_rules.tsv 42 TRAITS nextflow tsv stable - - +file {sample}/traits/{sample}_traits.tsv 42 TRAITS nextflow tsv stable - - +file {sample}/traits/{sample}_traits_genotypes.tsv 42 TRAITS_SITES nextflow tsv stable - - +file {sample}/vcf/{sample}.g.vcf.gz 03 DEEPVARIANT both vcf stable - - +file {sample}/vcf/{sample}.g.vcf.gz.tbi 03 DEEPVARIANT both index stable - - +file {sample}/vcf/{sample}.report.html 07 - bash html stable bash - +file {sample}/vcf/{sample}.report.json 07 - bash json stable bash - +file {sample}/vcf/{sample}.vcf.gz 03 DEEPVARIANT both vcf stable bash FORMAT/GT,FORMAT/GQ,FORMAT/DP +file {sample}/vcf/{sample}.vcf.gz.tbi 03 DEEPVARIANT both index stable bash - +file {sample}/vcf/{sample}_roh.txt 11 - bash text stable bash - +file {sample}/vcf/{sample}_roh_summary.txt 11 - bash tsv stable bash chrom,start,end,length_bp,length_mb +file {sample}/vcf_gatk/{sample}.vcf.gz 03a - bash vcf stable bash FORMAT/GT +file {sample}/vep/{sample}_annotated.vcf.gz 30 VCFANNO_INDEX both vcf stable both INFO/REVEL +file {sample}/vep/{sample}_annotated.vcf.gz.tbi 30 VCFANNO_INDEX nextflow index stable nextflow - +file {sample}/vep/{sample}_vep.vcf.gz 13 VEP both vcf stable - - +file {sample}/vep/{sample}_vep.vcf.gz.tbi 13 VEP nextflow index stable - - +file {sample}/vep/{sample}_vep.vcf_summary.html 13 VEP nextflow html stable - - +file {sample}/viral/{sample}_human_coverage.tsv 47 VIRAL_READS nextflow tsv experimental - - +file {sample}/viral/{sample}_viral.tsv 47 VIRAL_REPORT nextflow tsv experimental - - +file {sample}/viral/{sample}_viral_coverage.tsv 47 VIRAL_READS nextflow tsv experimental - - +file {sample}/viral/{sample}_viral_depth.tsv 47 VIRAL_READS nextflow tsv experimental - - +file {sample}/viral/{sample}_viral_reads.bam 47 VIRAL_READS nextflow bam experimental - - +file {sample}/viral/{sample}_viral_reads.bam.bai 47 VIRAL_READS nextflow index experimental - - +file {sample}/vntyper/run 52 VNTYPER nextflow dir experimental - - +file {sample}/vntyper/{sample}_vntyper.log 52 VNTYPER nextflow text experimental - - +file {sample}/vntyper/{sample}_vntyper.tsv 52 VNTYPER nextflow tsv experimental - - +file {sample}/y_haplogroup/{sample}_chrY.bam 37 Y_PILEUP nextflow bam experimental - - +file {sample}/y_haplogroup/{sample}_chrY.bam.bai 37 Y_PILEUP nextflow index experimental - - +file {sample}/y_haplogroup/{sample}_idxstats.txt 37 Y_PILEUP nextflow text internal - - +file {sample}/y_haplogroup/{sample}_y_haplogroup.txt 37 Y_HAPLOGROUP nextflow text experimental - - +file {sample}/y_haplogroup/{sample}_y_pileup.txt 37 Y_PILEUP nextflow text internal - - +file {sample}/y_haplogroup/{sample}_y_private_snps.tsv 37 Y_PRIVATE nextflow tsv experimental - - +file {sample}/y_haplogroup/{sample}_y_tree.txt 37 Y_HAPLOGROUP nextflow text experimental - - +file {sample}/y_haplogroup/{sample}_y_tree_path.tsv 37 Y_HAPLOGROUP nextflow tsv experimental - - +file {sample}/y_haplogroup/{sample}_y_tree_pileup.txt 37 Y_PILEUP nextflow text internal - - +file {sample}/y_str/{sample}_y_str.csv 45 Y_STR nextflow csv experimental - - +file {sample}/y_str/{sample}_y_str.log 45 Y_STR nextflow text experimental - - +file {sample}/y_str/{sample}_y_str.tsv 45 Y_STR nextflow tsv experimental - - +file {sample}/y_str/{sample}_y_str.vcf.gz 45 Y_STR nextflow vcf experimental - - +file {sample}/{sample}_charts.json 24 HTML_REPORT nextflow json stable - - +file {sample}/{sample}_report.html 24 HTML_REPORT both html stable bash - +file {sample}/{sample}_report.txt 24 - both text stable bash - +file {sample}/{sample}_search_index.json 24 HTML_REPORT nextflow json stable - - +file {sample}/{sample}_sex_check.tsv 16 INDEXCOV nextflow tsv stable - - +param input - - - - stable - - +param outdir - - - - stable - - +param allow_unfiltered - - - - stable - - +param publish_dir_mode - - - - stable - - +param skip_trim - - - - stable - - +param minimap2_index - - - - stable - - +param intervals - - - - stable - - +param sex_check - - - - stable - - +param reference - - - - stable - - +param clinvar - - - - stable - - +param clinvar_index - - - - stable - - +param tools - - - - stable - - +param vep_cache - - - - stable - - +param vep_cache_cpsr - - - - stable - - +param pcgr_data - - - - stable - - +param pypgx_bundle - - - - stable - - +param cadd_snv - - - - stable - - +param cadd_snv_index - - - - stable - - +param cadd_indel - - - - stable - - +param cadd_indel_index - - - - stable - - +param spliceai_snv - - - - stable - - +param spliceai_snv_index - - - - stable - - +param spliceai_indel - - - - stable - - +param spliceai_indel_index - - - - stable - - +param revel - - - - stable - - +param revel_index - - - - stable - - +param alphamissense - - - - stable - - +param alphamissense_index - - - - stable - - +param gnomad_constraint - - - - stable - - +param pgs_scoring - - - - stable - - +param ancestry_ref - - - - stable - - +param ancestry_fine_panel - - - - experimental - - +param ancient_panel - - - - experimental - - +param pgsc_calc - - - - stable - - +param prs_keep_ambiguous - - - - stable - - +param expansion_catalog - - - - stable - - +param hla_dat - - - - stable - - +param hla_genes - - - - stable - - +param gencode_exons - - - - stable - - +param mito_clinvar - - - - stable - - +param gnomad_mito - - - - stable - - +param kir - - - - experimental - - +param kir_dat - - - - experimental - - +param hla_qc - - - - experimental - - +param cyrius_install - - - - stable - - +param parascopy_data - - - - experimental - - +param yleaf_data - - - - experimental - - +param hpo - - - - experimental - - +param exomiser_data - - - - experimental - - +param family_name - - - - experimental - - +param exomiser_proband - - - - experimental - - +param exomiser_assembly - - - - experimental - - +param medications - - - - experimental - - +param archaic_data - - - - experimental - - +param archaic_pop - - - - experimental - - +param consumer_sites - - - - experimental - - +param megane_kmers - - - - experimental - - +param mocha_data - - - - experimental - - +param locityper_data - - - - experimental - - +param locityper_bg_region - - - - experimental - - +param viral_ref - - - - experimental - - +param chip_gff - - - - experimental - - +param chip_germline_resource - - - - experimental - - +param chip_pon - - - - experimental - - +param parascopy_population - - - - experimental - - +param parascopy_depth_bed - - - - experimental - - +param paraphase_regions - - - - experimental - - +param slivar_bin - - - - stable - - +param cnvpytor_resources - - - - stable - - +param annotsv_annotations - - - - stable - - +param delly_exclude - - - - stable - - +param manta_call_regions - - - - stable - - +param cytoband - - - - stable - - +param somalier_sites - - - - experimental - - +param verifybamid2_panel - - - - experimental - - +param freemix_warn - - - - experimental - - +param max_cpus - - - - stable - - +param max_memory - - - - stable - - +param max_time - - - - stable - - +tool pharmcat 07 - - - stable - - +tool cpic 27 - - - stable - - +tool clinvar 06 - - - stable - - +tool pypgx 32 - - - stable - - +tool vep 13 - - - stable - - +tool vcfanno 30 - - - stable - - +tool slivar 31 - - - stable - - +tool clinical_filter 23 - - - stable - - +tool cpsr 17 - - - stable - - +tool roh 11 - - - stable - - +tool prs 25 - - - stable - - +tool ancestry 26 - - - stable - - +tool ancestry_fine 26b - - - experimental - - +tool mito_haplogroup 12 - - - stable - - +tool hla_typing 08 - - - stable - - +tool expansion_hunter 09 - - - stable - - +tool stranger 09b - - - stable - - +tool telomere_hunter 10 - - - stable - - +tool mosdepth 16b - - - stable - - +tool mito_variants 20 - - - stable - - +tool cyrius 21 - - - stable - - +tool manta 04 - - - stable - - +tool delly 19 - - - stable - - +tool cnvpytor 18 - - - stable - - +tool duphold 15 - - - stable - - +tool annotsv 05 - - - stable - - +tool survivor_merge 22 - - - stable - - +tool sample_qc 33 - - - experimental - - +tool cram_archive 34 - - - experimental - - +tool parascopy 35 - - - experimental - - +tool paraphase 35b - - - experimental - - +tool y_haplogroup 37 - - - experimental - - +tool locityper 53 - - - experimental - - +tool html_report 24 - - - stable - - +tool multiqc 28 - - - stable - - +tool consumer_export 40 - - - experimental - - +tool bloodtype 41 - - - experimental - - +tool lpa 46 - - - experimental - - +tool vntyper 52 - - - experimental - - +tool traits 42 - - - stable - - +tool archaic 43 - - - experimental - - +tool ancient 44 - - - experimental - - +tool y_str 45 - - - experimental - - +tool viral 47 - - - experimental - - +tool exomiser 48 - - - experimental - - +tool chip 49 - - - experimental - - +tool mei 50 - - - experimental - - +tool mocha 51 - - - experimental - - +switch GENOME_DIR - - - - stable - - +switch SKIP_VALIDATION - - - - stable - - +switch THREADS - - - - stable - - +switch SKIP_TRIM - - - - stable - - +switch INTERVALS - - - - stable - - +switch ALIGN_DIR - - - - stable - - +switch TOOLS - - - - stable - - +switch KIR - - - - experimental - - +switch HLA_QC - - - - experimental - - +switch HPO - - - - experimental - - +switch REF_FASTA - - - - stable - - +switch EH_CATALOG - - - - stable - - +switch MANTA_CALL_REGIONS - - - - stable - - +switch PARASCOPY_POPULATION - - - - experimental - - +switch PARAPHASE_REGIONS - - - - experimental - - +switch ANCESTRY_PANEL - - - - stable - - +switch ANCESTRY_FINE_DIR - - - - experimental - - +switch ARCHAIC_DIR - - - - experimental - - +switch ARCHAIC_POP - - - - experimental - - +switch PGSC_CALC_DIR - - - - stable - - +switch ANCIENT_PANEL_DIR - - - - experimental - - +switch VIRAL_REF - - - - experimental - - +switch MEGANE_KMERS - - - - experimental - - +switch MOCHA_DATA - - - - experimental - - +switch GRIDSS - - - - stable - - +switch IMPUTATION - - - - stable - - +switch SOMATIC - - - - stable - - +switch EXTRA_CALLERS - - - - stable - - +switch BENCHMARK - - - - stable - - +switch FAMILIES - - - - experimental - - +switch OWNGENOME_LIVE_URL - - - - experimental - - +switch NXF_VER - - - - stable - - +switch NXF_WORK - - - - stable - - +switch NXF_HOME - - - - stable - - +switch NXF_OFFLINE - - - - stable - - diff --git a/docs/index.md b/docs/index.md index 484af88c..b462a9c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -128,6 +128,7 @@ graph LR - [Variant caller benchmarking](benchmarking.md): concordance and truth-set runs for the alternative callers. - [Common issues and FAQ](faq.md), then the [Troubleshooting guide](troubleshooting.md): symptom, cause, fix, per step. - [Lessons learned](lessons-learned.md): what failed during development and why the defaults are what they are. +- [What 1.0 keeps stable](output-contract.md): the options, config keys and result files a 1.x release never breaks, and how CI checks them. - [Tool selection rationale](tool-rationale.md), the [Glossary](glossary.md) and [Recommended resources](resources.md) for learning genomics. ## Getting help diff --git a/docs/output-contract.md b/docs/output-contract.md new file mode 100644 index 00000000..0d49b17c --- /dev/null +++ b/docs/output-contract.md @@ -0,0 +1,52 @@ +# What 1.0 Keeps Stable + +OwnGenome 1.0 makes one promise to anyone who scripts around it: the command options, the config keys and the names and formats of the result files do not break in a 1.x release. A break waits for 2.0. This page says what the promise covers, what counts as a break, and how CI holds it. + +The full list is one file, [`assets/output_contract.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/output_contract.tsv). Every row there is either `stable`, `experimental` or `internal`. + +## What is covered + +- **Nextflow params:** every param in `nextflow_schema.json` (`--input`, `--outdir`, `--tools`, `--clinvar`, ...), by name and meaning. +- **`--tools` names:** every name `main.nf` accepts (`pharmcat`, `clinvar`, `vcfanno`, ...). +- **`run-all.sh`:** its two arguments (` `) and every switch in its `SWITCHES` list (`TOOLS`, `THREADS`, `SKIP_TRIM`, `INTERVALS`, `REF_FASTA`, ...). +- **Result files:** every file the Nextflow pipeline publishes, which is also what `run-all.sh` writes, at its path under `GENOME_DIR//` (or `--outdir`). For each one the contract gives its step, its format and the fields a reader can count on: the columns of a table, the INFO and FORMAT fields of a VCF header, the top-level keys of a JSON file (`summary.json`'s keys are the ones its schema, `tests/schema/summary.schema.json`, requires). +- **Single-step scripts:** the files listed with leg `bash` or `both`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet. + +## What counts as a break + +A 1.x release never does any of these to a `stable` row: + +- remove or rename a file, or move it to another folder; +- remove or rename a column of a table, an INFO or FORMAT field of a VCF, or a top-level key of a JSON file; +- remove or rename a param, a `--tools` name or a `run-all.sh` switch, or change what it means. + +These are not breaks, and can come in any 1.x release: + +- a new file, column, field, key, param, tool or switch; +- a new row in a table, or a different value where the science changed: a newer ClinVar gives other hits, a better caller gives other calls. The contract is about the shape of the results, not about their content; +- a change to an `experimental` or `internal` row. + +When a 1.x change touches an output on purpose, its pull request says so, and we add a column or a file rather than rename one. + +## Stable, experimental and internal + +- **`stable`:** kept until 2.0. +- **`experimental`:** an opt-in engine that is new in 1.0 and has not been run by many people yet: Y haplogroups and Y-STRs, Parascopy and Paraphase, blood groups, Lp(a), VNtyper, viral DNA, CHIP, MEI, MoChA, Exomiser, archaic and ancient ancestry, fine-grained ancestry, sample identity (somalier and VerifyBamID2), the CRAM archive, the family and couple steps, KIR and OptiType. Their files, params, tools and switches are listed, so you can see what exists, but a 1.x release may change them. Each change is named in the release notes. An experimental engine becomes stable in a 1.x release by changing its rows; it never goes the other way. +- **`internal`:** a task file that the pipeline does not publish (a temporary VCF, a work index). It is listed so the list stays complete, and it promises nothing. + +The engines that run by default, and every step that existed before 1.0, are `stable`. + +## How CI holds it + +Two checks read the contract, both with [`scripts/ci/check-output-contract.py`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/check-output-contract.py): + +- **On every pull request, with no run** (`tests/test_output_contract.py`, in the `unit-tests` job of the Guard workflow): every param of `nextflow_schema.json`, every `--tools` name of `main.nf` and every switch of `run-all.sh` has a row, and every row still names one. Every file row of a Nextflow process still matches a file that process writes and publishes, and every file a module publishes has a row. A removed or renamed stable item fails as `BREAKING`. A new one with no row fails too, with the message to add it, so the contract is always the full list. The test also plants each kind of break on a copy of the code (a renamed output, a removed param, a removed switch, a new tool and a new output with no row) and requires each to fail. +- **On the HG002 end-to-end run** (`tests/e2e/output-contract-1-hg002.sh`): every stable file whose `e2e` column names a leg must be in the results that leg wrote, under its contracted name, with its required fields. The bash leg is the single-step scripts the base cases run; the Nextflow leg is case `60-nextflow`. Rows whose step the fixture run does not reach have `e2e` set to `-` and are held by the static check only. The two files `run-all.sh` writes itself, `run_manifest.tsv` and `logs/run_status.tsv`, are listed but no check holds them yet: neither the fixture run nor a module writes them. The same case copies the Nextflow results, drops the `clnsig` column of the ClinVar hits table and renames the ROH summary, and requires the check to fail on both. It also prints every file each leg wrote with its fields, which is what to read when the contract is written again. + +## Changing the contract + +- **A new output, param, tool or switch:** add its row in the same pull request. Make it `experimental` when it belongs to a new opt-in engine, `stable` otherwise. +- **A break that cannot wait for 2.0:** keep the old name beside the new one for the rest of 1.x, with both rows `stable`, say on the step's doc page that the old name is deprecated, and say it in the pull request and the release notes. +- **Promoting an engine:** change its rows from `experimental` to `stable` and name it in the release notes. + +[Testing](testing.md) has how CI runs; [Pipeline overview](pipeline-overview.md) has what each step is for. diff --git a/docs/testing.md b/docs/testing.md index f4040791..e6eca850 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -121,6 +121,8 @@ What it covers: - **Index times** (`run-all-sample-qc-and-stale-index-1-index-times.sh`): every index the direct `nextflow run` and the launcher published is not older than its data file, and samtools and bcftools print no stale-index warning on the BAM and the VCF. Then a short VCF-only run with three planted pairs checks `main.nf`'s completion handler: it sets the index times of the run's own files after the last copy, and leaves an index older than the run as it was. A copied pair with its index set an hour older is the negative control. - **The Nextflow report's tables** (`paralogs-report-section-1-nf-report-tables.sh`): `HTML_REPORT` runs for real on the CYP2D6 depth check, the cDNA read check and the phased compound-het pairs that earlier cases wrote, and its reports must show the depth-check line, the cDNA column with its flagged count and the pairs by phase; the same run without the three tables must show none of them. A stub run of every tool then checks that `main.nf` passes the cDNA table to each `HTML_REPORT` task that got ClinVar hits, against a copy of the pipeline with that join removed. +- **The 1.0 output contract** (`output-contract-1-hg002.sh`): every stable file of [`assets/output_contract.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/output_contract.tsv) whose `e2e` column names a leg is in what the base cases' bash steps and case `60-nextflow` wrote, under its contracted name, with its required columns, VCF fields and JSON keys. Its red control drops a column and renames a file on a copy and must fail on both. `tests/test_output_contract.py` checks the same contract against the code with no run ([What 1.0 keeps stable](output-contract.md)). + - **Report assets:** a last case lists every external `http(s)` address in a `src=` or `href=` attribute of the generated HTML reports, so the remote files a report loads when opened are known. The list goes to the job summary. Each check is a count or a column: a VCF with records, the sample name in the VCF header, a gene in the ClinVar hit, at least one called gene in PharmCAT's `report.json`, a CYP2D6 row from pypgx, the planted ClinVar row in the HTML report with its gene, its significance and no empty cell. An exit code alone never passes a case. diff --git a/scripts/ci/check-output-contract.py b/scripts/ci/check-output-contract.py new file mode 100755 index 00000000..e397cad5 --- /dev/null +++ b/scripts/ci/check-output-contract.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Check the 1.0 output contract, assets/output_contract.tsv (docs/output-contract.md). + +Two checks: + + --static [--repo DIR] + The command surface and the Nextflow file list against the code, with no + run: every param of nextflow_schema.json, every --tools name of main.nf + (known_tools) and every run-all.sh switch (its SWITCHES list) has a row, + and every row still names one; every file row of a process names a file + that process still writes and publishes; and every file a module + publishes has a row. A stable row that no longer matches is a break. A + new param, tool, switch or published file with no row fails too, so the + contract stays the full list. + + --outputs DIR --leg bash|nextflow --sample S [--observe] + The files a run wrote. DIR is the folder that holds / (GENOME_DIR + for the bash steps, --outdir for Nextflow). For every stable row whose e2e + column names this leg: the file must be there under its contracted name, + and its required columns (tsv, csv), INFO/FORMAT fields (vcf) or top-level + keys (json) must be in it. Experimental rows are listed, never enforced. + --observe also prints every file under DIR/ with what it has, the + lines to read when the contract is written again. + +Exit 0 when nothing is broken, 1 when something is, 2 on a usage error. Only +the standard library: it runs on the CI host and in self-tests. +""" +import argparse +import fnmatch +import glob +import gzip +import json +import os +import re +import sys + +COLUMNS = ["kind", "name", "step", "process", "legs", "format", "stability", "e2e", "required"] +KINDS = {"file", "param", "tool", "switch"} +STABILITY = {"stable", "experimental", "internal"} +LEGS = {"bash", "nextflow", "both"} +FORMATS = {"tsv", "csv", "vcf", "json", "text", "html", "bam", "cram", "index", "bed", "dir", + "zip", "svg", "png", "md", "yml", "fasta", "bin"} + + +def read_contract(path): + """The rows of the contract as dicts; raises ValueError on a malformed file.""" + rows, header = [], None + with open(path) as f: + for n, line in enumerate(f, 1): + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + cells = line.split("\t") + if header is None: + if cells != COLUMNS: + raise ValueError(f"{path}:{n}: header must be {' '.join(COLUMNS)}, got {' '.join(cells)}") + header = cells + continue + if len(cells) != len(COLUMNS): + raise ValueError(f"{path}:{n}: {len(cells)} cells, want {len(COLUMNS)}") + r = dict(zip(COLUMNS, cells)) + r["line"] = n + bad = [] + if r["kind"] not in KINDS: + bad.append(f"kind {r['kind']}") + if r["stability"] not in STABILITY: + bad.append(f"stability {r['stability']}") + if r["kind"] == "file": + if r["legs"] not in LEGS: + bad.append(f"legs {r['legs']}") + if r["format"] not in FORMATS: + bad.append(f"format {r['format']}") + if r["e2e"] not in ("-", "bash", "nextflow", "both"): + bad.append(f"e2e {r['e2e']}") + if r["legs"] == "bash" and r["process"] != "-": + bad.append("a bash-only row names a process") + if bad: + raise ValueError(f"{path}:{n}: bad " + ", ".join(bad)) + rows.append(r) + if header is None: + raise ValueError(f"{path}: no header line") + seen = {} + for r in rows: + key = (r["kind"], r["name"], r["legs"] if r["kind"] == "file" else "") + if key in seen: + raise ValueError(f"{path}:{r['line']}: {r['kind']} {r['name']} is listed twice (line {seen[key]})") + seen[key] = r["line"] + return rows + + +# ---------------------------------------------------------------- the code side + +def braces(pattern): + """Expand one level of {a,b} alternatives, as Nextflow globs do.""" + m = re.search(r"\{([^{}]*)\}", pattern) + if not m: + return [pattern] + out = [] + for alt in m.group(1).split(","): + out.extend(braces(pattern[:m.start()] + alt + pattern[m.end():])) + return out + + +def concrete(pattern, sample="SAMPLE"): + """A Nextflow path with ${meta.id} and the family name filled in.""" + p = pattern.replace("${meta.id}", sample).replace("{sample}", sample) + p = re.sub(r"\$\{params\.family_name[^}]*\}", "FAMILY", p).replace("{family}", "FAMILY") + p = p.replace("${params.outdir}/", "").replace("${params.outdir}", "") + return p.strip("/") + + +def module_processes(repo): + """{process: {'publish': [(dir, [patterns] or None, has_saveas)], 'outputs': [glob]}} of modules/local.""" + procs = {} + for f in sorted(glob.glob(os.path.join(repo, "modules", "local", "*", "main.nf"))): + src = open(f).read() + starts = [(m.start(), m.group(1)) for m in re.finditer(r"^process\s+(\w+)\s*\{", src, re.M)] + for i, (pos, name) in enumerate(starts): + body = src[pos:starts[i + 1][0] if i + 1 < len(starts) else len(src)] + head = re.split(r"^\s*(?:script|shell|exec|stub):", body, maxsplit=1, flags=re.M)[0] + pubs = [] + lines = head.split("\n") + for j, line in enumerate(lines): + if not re.match(r"\s*publishDir\b", line): + continue + stmt = line + k = j + while stmt.rstrip().endswith(",") or stmt.count("{") > stmt.count("}"): + k += 1 + if k >= len(lines): + break + stmt += "\n" + lines[k] + d = re.search(r'publishDir\s+\{?\s*"([^"]+)"', stmt) + pat = re.search(r"pattern:\s*[\"']([^\"']+)[\"']", stmt) + pubs.append((concrete(d.group(1)) if d else "", + braces(pat.group(1)) if pat else None, + "saveAs" in stmt)) + out = re.search(r"^\s*output:\s*$(.*?)(?:^\s*(?:when|topic):|\Z)", head, re.M | re.S) + outputs = [] + if out: + for a, b in re.findall(r"path\s*\(?\s*\"([^\"]+)\"|path\s*\(?\s*'([^']+)'", out.group(1)): + g = a or b + if g != "versions.yml": + outputs.append(g) + procs[name] = {"publish": pubs, "outputs": outputs, "file": os.path.relpath(f, repo)} + return procs + + +def published(proc): + """(contracted-path glob, basename-only) pairs a process publishes; basename-only when a saveAs renames.""" + items = [] + for d, pats, saveas in proc["publish"]: + for g in proc["outputs"]: + c = concrete(g) + if pats is not None and not any(fnmatch.fnmatch(c.replace("*", "X"), p) or fnmatch.fnmatch(c, p) + for p in pats): + continue + items.append((g, (d + "/" + c) if d else c, saveas)) + return items + + +def schema_params(repo): + s = json.load(open(os.path.join(repo, "nextflow_schema.json"))) + defs = s.get("definitions") or s.get("$defs") or {} + return {p for group in defs.values() for p in group.get("properties", {})} + + +def known_tools(repo): + src = open(os.path.join(repo, "main.nf")).read() + m = re.search(r"def known_tools = \[(.*?)\]", src, re.S) + return set(re.findall(r"'([a-z0-9_]+)'", m.group(1))) if m else set() + + +def run_all_switches(repo): + src = open(os.path.join(repo, "scripts", "run-all.sh")).read() + m = re.search(r"^SWITCHES=\((.*?)\)", src, re.S | re.M) + return set(m.group(1).split()) if m else set() + + +def check_static(repo, rows): + problems = [] + have = {"param": schema_params(repo), "tool": known_tools(repo), "switch": run_all_switches(repo)} + where = {"param": "nextflow_schema.json", "tool": "main.nf known_tools", "switch": "scripts/run-all.sh SWITCHES"} + for kind in ("param", "tool", "switch"): + if not have[kind]: + problems.append(f"found no {kind}s in {where[kind]}: the reader no longer matches the code") + continue + listed = {r["name"]: r for r in rows if r["kind"] == kind} + for name, r in sorted(listed.items()): + if name not in have[kind]: + what = "BREAKING: the stable" if r["stability"] == "stable" else "the" + problems.append(f"{what} {kind} {name} (contract line {r['line']}) is gone from {where[kind]}") + for name in sorted(have[kind] - set(listed)): + problems.append(f"{kind} {name} of {where[kind]} has no row in the contract: add one") + + procs = module_processes(repo) + if not procs: + problems.append("found no processes under modules/local") + file_rows = [r for r in rows if r["kind"] == "file" and r["process"] != "-"] + for r in file_rows: + if r["process"] not in procs: + what = "BREAKING: " if r["stability"] == "stable" else "" + problems.append(f"{what}{r['name']} (contract line {r['line']}): process {r['process']} is not in modules/local") + continue + p = procs[r["process"]] + want = concrete(r["name"]) + if r["stability"] == "internal": + ok = any(fnmatch.fnmatch(os.path.basename(want), os.path.basename(concrete(g))) for g in p["outputs"]) + else: + ok = any(fnmatch.fnmatch(want, path) if not saveas + else fnmatch.fnmatch(os.path.basename(want), os.path.basename(path)) + for _, path, saveas in published(p)) + if not ok: + what = "BREAKING: the stable file" if r["stability"] == "stable" else "the file" + problems.append(f"{what} {r['name']} (contract line {r['line']}) is not what {r['process']} " + f"publishes ({p['file']}): renamed, moved or no longer written") + for name, p in sorted(procs.items()): + rows_p = [concrete(r["name"]) for r in file_rows if r["process"] == name] + if not p["publish"]: + continue + for g in p["outputs"]: + c = os.path.basename(concrete(g)) + if not any(fnmatch.fnmatch(os.path.basename(w), c) for w in rows_p): + problems.append(f"{name} ({p['file']}) writes {g} and has no contract row for it: " + "add one (internal when it is not published)") + return problems + + +# ---------------------------------------------------------------- the run side + +def opener(path): + with open(path, "rb") as f: + magic = f.read(2) + if magic == b"\x1f\x8b": + return gzip.open(path, "rt", errors="replace") + return open(path, errors="replace") + + +def fields(path, fmt): + """What a file has: its columns, its INFO/FORMAT ids or its top-level keys.""" + if fmt in ("tsv", "csv"): + sep = "\t" if fmt == "tsv" else "," + with opener(path) as f: + for i, line in enumerate(f): + if i > 200: + break + line = line.rstrip("\n") + if line.startswith("##") or sep not in line: + continue + return [c.strip() for c in line.lstrip("#").split(sep)] + return [] + if fmt == "vcf": + out = [] + with opener(path) as f: + for line in f: + if not line.startswith("##"): + break + m = re.match(r"##(INFO|FORMAT|FILTER)=]+)", line) + if m: + out.append(f"{m.group(1)}/{m.group(2)}") + return out + if fmt == "json": + try: + data = json.load(opener(path)) + except ValueError: + return None + return list(data.keys()) if isinstance(data, dict) else [] + return [] + + +def guess_format(path): + b = os.path.basename(path) + for ext, fmt in ((".vcf.gz", "vcf"), (".vcf", "vcf"), (".tsv.gz", "tsv"), (".tsv", "tsv"), (".csv", "csv"), + (".json", "json"), (".txt", "tsv")): + if b.endswith(ext): + return fmt + return "-" + + +def check_outputs(rows, outdir, leg, sample, observe): + problems, checked = [], 0 + for r in rows: + if r["kind"] != "file" or r["stability"] != "stable" or r["e2e"] not in (leg, "both"): + continue + if r["legs"] not in (leg, "both"): + problems.append(f"contract line {r['line']}: e2e {r['e2e']} on a {r['legs']} row") + continue + checked += 1 + name = r["name"].replace("{sample}", sample) + hits = sorted(glob.glob(os.path.join(outdir, name))) + if not hits: + problems.append(f"BREAKING: the stable file {name} (contract line {r['line']}, step {r['step']}) " + f"is missing from {outdir}: renamed, moved or not written") + continue + req = [] if r["required"] == "-" else r["required"].split(",") + if r["format"] == "dir": + if not all(os.path.isdir(h) for h in hits): + problems.append(f"{name} (contract line {r['line']}) is not a folder") + continue + for h in hits: + if os.path.isdir(h): + problems.append(f"{os.path.relpath(h, outdir)} (contract line {r['line']}) is a folder, want a file") + continue + if not req: + continue + have = fields(h, r["format"]) + if have is None: + problems.append(f"{os.path.relpath(h, outdir)} (contract line {r['line']}) is not valid {r['format']}") + continue + for col in req: + if col not in have: + problems.append(f"BREAKING: {os.path.relpath(h, outdir)} (contract line {r['line']}) " + f"has no {col} (it has: {','.join(have) or 'nothing'})") + if observe: + base = os.path.join(outdir, sample) + print(f"# observed under {base} (leg {leg}): path, format, fields") + for root, dirs, files in os.walk(base): + dirs[:] = sorted(d for d in dirs if d not in ("work", "nf-work", ".nextflow")) + depth = os.path.relpath(root, base).count(os.sep) + for fn in sorted(files): + p = os.path.join(root, fn) + rel = os.path.relpath(p, outdir).replace(sample, "{sample}") + fmt = guess_format(p) + try: + have = fields(p, fmt) if fmt != "-" and os.path.getsize(p) < 2 << 30 else [] + except (OSError, EOFError, UnicodeDecodeError): + have = [] + print(f"OBSERVED\t{leg}\t{rel}\t{fmt}\t{','.join(have or [])}"[:4000]) + if depth >= 3: + dirs[:] = [] + return checked, problems + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--contract", help="default: /assets/output_contract.tsv") + ap.add_argument("--repo", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + ap.add_argument("--static", action="store_true") + ap.add_argument("--outputs") + ap.add_argument("--leg", choices=["bash", "nextflow"]) + ap.add_argument("--sample") + ap.add_argument("--observe", action="store_true") + a = ap.parse_args(argv) + if not a.static and not a.outputs: + ap.error("give --static or --outputs") + if a.outputs and not (a.leg and a.sample): + ap.error("--outputs needs --leg and --sample") + try: + rows = read_contract(a.contract or os.path.join(a.repo, "assets", "output_contract.tsv")) + except (OSError, ValueError) as e: + print(f"ERROR: {e}") + return 1 + problems = [] + if a.static: + p = check_static(a.repo, rows) + print(f"static: {len(rows)} contract rows, {len(p)} problems") + problems += p + if a.outputs: + n, p = check_outputs(rows, a.outputs, a.leg, a.sample, a.observe) + print(f"outputs ({a.leg}): {n} stable files checked under {a.outputs}, {len(p)} problems") + if n == 0: + p.append(f"no stable row has e2e {a.leg}: the check checked nothing") + problems += p + for p in problems: + print(f"FAIL {p}") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 3bf941d0..25230284 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -109,6 +109,7 @@ nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.ht nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/manta_inv_filter.py bin/sv_svlen_fix.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/copy_number.py bin/cyp2d6_depth_check.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/manta_inv_filter.py bin/pgx_mito_merge.py bin/pgx_outside_calls.py bin/pgx_refblock_fill.py bin/render_report.py bin/results_index.py bin/sv_svlen_fix.py bin/y_tree_place.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cdna_scan.py bin/clinvar_hits.awk bin/clinvar_join_adjacent.awk bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/comphet_phase.py bin/constraint_join.awk bin/copy_number.py bin/csq_rank.awk bin/cyp2d6_depth_check.py bin/export_consumer.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/rfc1_motif.py bin/strchive_convert.py bin/sv_svlen_fix.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +output-contract-1-hg002 assets/* bin/* modules/* scripts/* paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/copy_number.py bin/hba_copy_number.py bin/lpa_kiv2.py bin/paraphase_report.py bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 96e74ca1..036d76a7 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -108,6 +108,7 @@ nextflow-from-fastq-3-sexcheck shard 97 - nextflow-hardening-1-manta shard 21 - nextflow-hardening-2-nf shard 643 bash-step-19-delly,nextflow-hardening-1-manta nextflow-report-parity-1-sections shard 206 clinvar-clip-check-1,comphet-phasing-1-reads +output-contract-1-hg002 shard 30 - paralog-catalog-1-gba shard 114 - paralog-checks-and-doc-fixes-1-hba shard 57 - paralog-rows-1 shard 316 - diff --git a/tests/e2e/output-contract-1-hg002.sh b/tests/e2e/output-contract-1-hg002.sh new file mode 100755 index 00000000..b42aadca --- /dev/null +++ b/tests/e2e/output-contract-1-hg002.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# The 1.0 output contract (assets/output_contract.tsv) on the HG002 base outputs: every stable file +# the bash steps and the Nextflow run (case 60) wrote is there under its contracted name, with its +# required columns, VCF fields and JSON keys. Red control: a copy of the Nextflow results with one +# column dropped and one file renamed must fail, naming both. +. "$(dirname "$0")/lib.sh" + +CHECK="${REPO}/scripts/ci/check-output-contract.py" +NF_OUT="${GENOME_DIR}/nf-results" + +contract() { # contract : the run check, its output in ; returns its exit code + python3 "$CHECK" --outputs "$1" --leg "$2" --sample "$SAMPLE" --observe > "$3" 2>&1 +} + +contract "$GENOME_DIR" bash "${CASE_TMP}/bash.log"; RC_BASH=$? +grep -v '^OBSERVED' "${CASE_TMP}/bash.log" +check_eq "the bash steps' outputs keep the contract" "$RC_BASH" 0 +check_ge "stable bash files checked" "$(sed -n 's/^outputs (bash): \([0-9]*\) stable.*/\1/p' "${CASE_TMP}/bash.log")" 20 + +contract "$NF_OUT" nextflow "${CASE_TMP}/nf.log"; RC_NF=$? +grep -v '^OBSERVED' "${CASE_TMP}/nf.log" +check_eq "the Nextflow outputs keep the contract" "$RC_NF" 0 +check_ge "stable Nextflow files checked" "$(sed -n 's/^outputs (nextflow): \([0-9]*\) stable.*/\1/p' "${CASE_TMP}/nf.log")" 10 + +# What each leg wrote, with its fields: the lines to read when the contract is written again. +echo "--- observed files (path, format, fields)" +grep -h '^OBSERVED' "${CASE_TMP}/bash.log" "${CASE_TMP}/nf.log" + +# Red control on a copy: the ClinVar hits table loses its clnsig column, the ROH summary is renamed. +PLANT="${CASE_TMP}/planted" +rm -rf "$PLANT"; mkdir -p "$PLANT" +cp -a "${NF_OUT}/${SAMPLE}" "${NF_OUT}/pipeline_info" "${PLANT}/" +HITS="${PLANT}/${SAMPLE}/clinvar/${SAMPLE}_clinvar_hits.tsv" +ROH="${PLANT}/${SAMPLE}/roh/${SAMPLE}_roh_summary.txt" +if [ -f "$HITS" ] && [ -f "$ROH" ]; then + awk -F'\t' -v OFS='\t' 'NR == 1 {for (i = 1; i <= NF; i++) if ($i == "clnsig") c = i} + {out = ""; for (i = 1; i <= NF; i++) if (i != c) out = out (out == "" ? "" : OFS) $i; print out}' \ + "$HITS" > "${HITS}.new" && mv "${HITS}.new" "$HITS" + mv "$ROH" "${ROH%_roh_summary.txt}_roh_segments.txt" +fi +contract "$PLANT" nextflow "${CASE_TMP}/planted.log"; RC_PLANT=$? +grep '^FAIL' "${CASE_TMP}/planted.log" +check_eq "the planted copy fails the contract" "$RC_PLANT" 1 +check_eq "the planted copy fails on the two plants only" "$(grep -c '^FAIL ' "${CASE_TMP}/planted.log")" 2 +check "the dropped clnsig column is named" grep -q "BREAKING: ${SAMPLE}/clinvar/${SAMPLE}_clinvar_hits.tsv .* has no clnsig" "${CASE_TMP}/planted.log" +check "the renamed ROH summary is named" grep -q "BREAKING: the stable file ${SAMPLE}/roh/${SAMPLE}_roh_summary.txt .* is missing" "${CASE_TMP}/planted.log" + +finish diff --git a/tests/test_output_contract.py b/tests/test_output_contract.py new file mode 100644 index 00000000..38c6de8a --- /dev/null +++ b/tests/test_output_contract.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""assets/output_contract.tsv and scripts/ci/check-output-contract.py. + + 1. the static check on this checkout passes: every param, --tools name, + run-all.sh switch and published Nextflow file has its row, and every row + still matches the code; + 2. static controls, each on a copy of the code: a renamed published file, a + removed param, a removed switch, a new --tools name with no row and a new + published file with no row are each refused, and the stable ones are + named BREAKING; + 3. the run check on a planted results folder made from the contract's own + e2e rows passes for both legs; then a dropped column, a dropped VCF + field, a dropped JSON key and a renamed file are each refused, naming the + file and what is missing; an experimental row is never enforced; + 4. a contract with a row listed twice or a bad stability is refused. + +Run: python3 tests/test_output_contract.py +""" +import gzip +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +CHECK = os.path.join(REPO, "scripts", "ci", "check-output-contract.py") +CONTRACT = os.path.join(REPO, "assets", "output_contract.tsv") +SAMPLE = "S1" +FAILS = 0 + + +def check(desc, ok, detail=""): + global FAILS + print(f"[{'PASS' if ok else 'FAIL'}] {desc}" + ("" if ok else f" -- {detail}")) + if not ok: + FAILS += 1 + + +def run(*args): + p = subprocess.run([sys.executable, CHECK, *args], capture_output=True, text=True) + return p.returncode, p.stdout + p.stderr + + +def rows(): + out, header = [], None + for line in open(CONTRACT): + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + cells = line.split("\t") + if header is None: + header = cells + continue + out.append(dict(zip(header, cells))) + return out + + +def code_copy(tmp): + """The files the static check reads, copied, so a control can change them.""" + dst = os.path.join(tmp, "repo") + shutil.copytree(os.path.join(REPO, "modules"), os.path.join(dst, "modules")) + os.makedirs(os.path.join(dst, "scripts")) + for f in ("main.nf", "nextflow_schema.json", os.path.join("scripts", "run-all.sh")): + shutil.copy(os.path.join(REPO, f), os.path.join(dst, f)) + return dst + + +def edit(path, old, new, count=1): + src = open(path).read() + assert old in src, f"{old!r} not in {path}" + open(path, "w").write(src.replace(old, new, count)) + + +def static_controls(): + rc, out = run("--static") + check("static check passes on this checkout", rc == 0 and "0 problems" in out, out[-2000:]) + + cases = [ + ("a renamed published file (ROH's summary)", + lambda d: edit(os.path.join(d, "modules", "local", "roh", "main.nf"), + 'path("*_roh_summary.txt")', 'path("*_roh_segments.txt")'), + [r"BREAKING: the stable file \{sample\}/roh/\{sample\}_roh_summary\.txt", r"writes \*_roh_segments\.txt and has no contract row"]), + ("a removed param (--intervals)", + lambda d: _drop_param(d, "intervals"), + [r"BREAKING: the stable param intervals"]), + ("a removed run-all.sh switch (SKIP_TRIM)", + lambda d: edit(os.path.join(d, "scripts", "run-all.sh"), " SKIP_TRIM ", " "), + [r"BREAKING: the stable switch SKIP_TRIM"]), + ("a new --tools name with no row", + lambda d: edit(os.path.join(d, "main.nf"), "'pharmcat', 'cpic',", "'pharmcat', 'newtool', 'cpic',"), + [r"tool newtool of main.nf known_tools has no row"]), + ("a new published file with no row", + lambda d: edit(os.path.join(d, "modules", "local", "roh", "main.nf"), + 'path("*_roh.txt"),', 'path("*_roh.txt"),\n tuple val(meta), path("*_roh_extra.tsv"), emit: extra\n'), + [r"ROH .* writes \*_roh_extra\.tsv and has no contract row"]), + ] + for desc, change, want in cases: + tmp = tempfile.mkdtemp() + try: + d = code_copy(tmp) + change(d) + rc, out = run("--static", "--repo", d, "--contract", CONTRACT) + missing = [w for w in want if not re.search(w, out)] + check(f"static control refused: {desc}", rc == 1 and not missing, f"rc={rc}, missing {missing}: {out[-1500:]}") + finally: + shutil.rmtree(tmp) + + +def _drop_param(d, name): + path = os.path.join(d, "nextflow_schema.json") + s = json.load(open(path)) + for group in s["definitions"].values(): + group.get("properties", {}).pop(name, None) + json.dump(s, open(path, "w")) + + +def plant(root, leg): + """A results folder with every stable e2e file of a leg, each carrying its required fields.""" + for r in rows(): + if r["kind"] != "file" or r["stability"] != "stable" or r["e2e"] not in (leg, "both"): + continue + path = os.path.join(root, r["name"].replace("{sample}", SAMPLE).replace("*", "x")) + req = [] if r["required"] == "-" else r["required"].split(",") + os.makedirs(os.path.dirname(path), exist_ok=True) + if r["format"] == "dir": + os.makedirs(path, exist_ok=True) + continue + if r["format"] == "vcf": + text = "##fileformat=VCFv4.2\n" + "".join( + f"##{f.split('/')[0]}=\n" for f in req) + \ + f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{SAMPLE}\n" + elif r["format"] == "json": + text = json.dumps({k: None for k in req}) + elif r["format"] in ("tsv", "csv") and req: + sep = "\t" if r["format"] == "tsv" else "," + text = "# a comment line\n" + sep.join(req) + "\n" + sep.join("1" for _ in req) + "\n" + else: + text = "x\n" + if path.endswith(".gz"): + with gzip.open(path, "wt") as f: + f.write(text) + else: + open(path, "w").write(text) + + +def run_controls(): + legs = {"bash": None, "nextflow": None} + tmp = tempfile.mkdtemp() + try: + for leg in legs: + root = os.path.join(tmp, leg) + plant(root, leg) + rc, out = run("--outputs", root, "--leg", leg, "--sample", SAMPLE) + n = re.search(r"(\d+) stable files checked", out) + check(f"run check passes on a planted {leg} folder", rc == 0 and n and int(n.group(1)) >= 5, out[-1500:]) + legs[leg] = root + + nf = legs["nextflow"] + hits = os.path.join(nf, SAMPLE, "clinvar", f"{SAMPLE}_clinvar_hits.tsv") + lines = open(hits).read().split("\n") + cols = lines[1].split("\t") + i = cols.index("clnsig") + open(hits, "w").write("\n".join("\t".join(c for j, c in enumerate(l.split("\t")) if j != i) for l in lines)) + rc, out = run("--outputs", nf, "--leg", "nextflow", "--sample", SAMPLE) + check("a dropped column (clnsig of the ClinVar hits) is refused and named", + rc == 1 and re.search(r"BREAKING: S1/clinvar/S1_clinvar_hits\.tsv .* has no clnsig", out), out[-1500:]) + + roh = os.path.join(nf, SAMPLE, "roh", f"{SAMPLE}_roh_summary.txt") + os.rename(roh, roh.replace("_roh_summary.txt", "_roh_segments.txt")) + rc, out = run("--outputs", nf, "--leg", "nextflow", "--sample", SAMPLE) + check("a renamed file (ROH summary) is refused as missing", + rc == 1 and re.search(r"BREAKING: the stable file S1/roh/S1_roh_summary\.txt .* is missing", out), out[-1500:]) + + ann = os.path.join(nf, SAMPLE, "vep", f"{SAMPLE}_annotated.vcf.gz") + with gzip.open(ann, "rt") as f: + text = f.read() + with gzip.open(ann, "wt") as f: + f.write(re.sub(r"##INFO=