From f3d1046f7740c8bf5d35bd6ac2b7a48c42ebcac4 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 17 Feb 2026 14:30:26 -0800 Subject: [PATCH 1/4] Add interproscan batch size --- main.nf | 1 + nextflow.config | 1 + nextflow_schema.json | 8 +++ .../local/functional_annotation/main.nf | 56 +++++++++++++++++-- workflows/proteinannotator.nf | 12 ++-- 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/main.nf b/main.nf index 98d7d67..64a3ced 100644 --- a/main.nf +++ b/main.nf @@ -49,6 +49,7 @@ workflow NFCORE_PROTEINANNOTATOR { params.skip_interproscan, params.interproscan_db_url, params.interproscan_db, + params.interproscan_batch_size, params.skip_s4pred ) emit: diff --git a/nextflow.config b/nextflow.config index e56f91f..43bf445 100644 --- a/nextflow.config +++ b/nextflow.config @@ -33,6 +33,7 @@ params { interproscan_db = null interproscan_applications = 'Hamap,PANTHER,PIRSF,TIGRFAM,sfld' interproscan_enableprecalc = false + interproscan_batch_size = 1000 // Secondary structure prediction (s4pred) skip_s4pred = false diff --git a/nextflow_schema.json b/nextflow_schema.json index b7ad6d8..10aaf42 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -320,6 +320,14 @@ "help_text": "This increases the speed of functional annotation with InterProScan by pre-calculating matches found in the UniProtKB, thereby identifying unique matches in the query sequences for faster annotation. By default this is turned off.\n\nFor more information about this flag see the tool [documentation](https://interproscan-docs.readthedocs.io/en/latest/HowToRun.html).\n\n> Modifies tool parameter(s):\n> - InterProScan: `---diasable-precalc`", "description": "Pre-calculates residue mutual matches.", "fa_icon": "fas fa-clock" + }, + "interproscan_batch_size": { + "type": "integer", + "default": 1000, + "minimum": 1, + "description": "Number of sequences per InterProScan batch.", + "help_text": "Split input FASTA files into batches of this many sequences before running InterProScan. This enables parallel processing of large proteomes and reduces memory usage per job. Results are automatically concatenated after all batches complete. Default: 1000 sequences per batch.", + "fa_icon": "fas fa-layer-group" } }, "help_text": "This subworkflow adds additional protein annotations to all input sequences. Currently, only annotation with InterProScan is integrated in the subworkflow.", diff --git a/subworkflows/local/functional_annotation/main.nf b/subworkflows/local/functional_annotation/main.nf index 240c504..720d2ba 100644 --- a/subworkflows/local/functional_annotation/main.nf +++ b/subworkflows/local/functional_annotation/main.nf @@ -2,12 +2,29 @@ include { ARIA2 } from '../../../modules/nf-core/aria2/main' include { UNTAR } from '../../../modules/nf-core/untar/main' include { INTERPROSCAN } from '../../../modules/nf-core/interproscan/main' +process CONCAT_TSV { + tag "$meta.id" + label 'process_single' + + input: + tuple val(meta), path(tsvs) + + output: + tuple val(meta), path("${meta.id}_interproscan.tsv"), emit: tsv + + script: + """ + cat ${tsvs} > ${meta.id}_interproscan.tsv + """ +} + workflow FUNCTIONAL_ANNOTATION { take: - ch_fasta // channel: [ val(meta), [ fasta ] ] - skip_interproscan // boolean - interproscan_db_url // string, url to download db - interproscan_db // string, existing db + ch_fasta // channel: [ val(meta), [ fasta ] ] + skip_interproscan // boolean + interproscan_db_url // string, url to download db + interproscan_db // string, existing db + interproscan_batch_size // integer, number of sequences per batch main: ch_interproscan_tsv = channel.empty() @@ -25,8 +42,35 @@ workflow FUNCTIONAL_ANNOTATION { ch_interproscan_db = UNTAR.out.untar.map{ f -> f[1] } } - INTERPROSCAN( ch_fasta, ch_interproscan_db ) - ch_interproscan_tsv = ch_interproscan_tsv.mix(INTERPROSCAN.out.tsv) + // Split FASTA into batches for parallel InterProScan processing + ch_fasta_batched = ch_fasta + .flatMap { meta, fasta -> + def chunks = fasta.splitFasta(by: interproscan_batch_size, file: true) + if (chunks instanceof Path) { + // Single chunk (fewer sequences than batch size) + return [[ meta, chunks ]] + } + chunks.withIndex().collect { chunk, idx -> + def new_meta = meta.clone() + new_meta.original_id = meta.id + new_meta.id = "${meta.id}_batch${idx}" + [ new_meta, chunk ] + } + } + + INTERPROSCAN( ch_fasta_batched, ch_interproscan_db ) + + // Regroup batch TSV results by original sample ID + ch_batched_tsv = INTERPROSCAN.out.tsv + .map { meta, tsv -> + def original_id = meta.original_id ?: meta.id + [ [id: original_id], tsv ] + } + .groupTuple() + + // Concatenate batch TSVs into one file per sample + CONCAT_TSV( ch_batched_tsv ) + ch_interproscan_tsv = CONCAT_TSV.out.tsv } emit: diff --git a/workflows/proteinannotator.nf b/workflows/proteinannotator.nf index fae1d7a..f84e356 100644 --- a/workflows/proteinannotator.nf +++ b/workflows/proteinannotator.nf @@ -29,10 +29,11 @@ workflow PROTEINANNOTATOR { skip_funfam // boolean funfam_db // string, path to the pfam HMM database, if already exists funfam_latest_link // string, path to the latest pfam HMM database, to download - skip_interproscan // boolean - interproscan_db_url // string, url to download db - interproscan_db // string, existing db - skip_s4pred // boolean + skip_interproscan // boolean + interproscan_db_url // string, url to download db + interproscan_db // string, existing db + interproscan_batch_size // integer, number of sequences per batch + skip_s4pred // boolean main: @@ -57,7 +58,8 @@ workflow PROTEINANNOTATOR { FAA_SEQFU_SEQKIT.out.fasta, skip_interproscan, interproscan_db_url, - interproscan_db + interproscan_db, + interproscan_batch_size ) ch_versions = ch_versions.mix( FUNCTIONAL_ANNOTATION.out.versions ) From 7781e7f90e4722a6c022f14704eb33eef9cb2c4b Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 17 Feb 2026 14:30:44 -0800 Subject: [PATCH 2/4] Remove concat tsv process --- .../local/functional_annotation/main.nf | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/subworkflows/local/functional_annotation/main.nf b/subworkflows/local/functional_annotation/main.nf index 720d2ba..e25f866 100644 --- a/subworkflows/local/functional_annotation/main.nf +++ b/subworkflows/local/functional_annotation/main.nf @@ -2,22 +2,6 @@ include { ARIA2 } from '../../../modules/nf-core/aria2/main' include { UNTAR } from '../../../modules/nf-core/untar/main' include { INTERPROSCAN } from '../../../modules/nf-core/interproscan/main' -process CONCAT_TSV { - tag "$meta.id" - label 'process_single' - - input: - tuple val(meta), path(tsvs) - - output: - tuple val(meta), path("${meta.id}_interproscan.tsv"), emit: tsv - - script: - """ - cat ${tsvs} > ${meta.id}_interproscan.tsv - """ -} - workflow FUNCTIONAL_ANNOTATION { take: ch_fasta // channel: [ val(meta), [ fasta ] ] @@ -59,18 +43,7 @@ workflow FUNCTIONAL_ANNOTATION { } INTERPROSCAN( ch_fasta_batched, ch_interproscan_db ) - - // Regroup batch TSV results by original sample ID - ch_batched_tsv = INTERPROSCAN.out.tsv - .map { meta, tsv -> - def original_id = meta.original_id ?: meta.id - [ [id: original_id], tsv ] - } - .groupTuple() - - // Concatenate batch TSVs into one file per sample - CONCAT_TSV( ch_batched_tsv ) - ch_interproscan_tsv = CONCAT_TSV.out.tsv + ch_interproscan_tsv = INTERPROSCAN.out.tsv } emit: From 1d0609cd13bb12fda31fb18c195a41e489a0b39e Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Sun, 22 Feb 2026 11:34:53 -0800 Subject: [PATCH 3/4] Update snapshot --- tests/default.nf.test.snap | 50 +++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 527d241..cccf3c4 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -80,21 +80,21 @@ "downloaded_dbs/interproscan_test.tar.gz", "functional_annotation", "functional_annotation/interproscan", - "functional_annotation/interproscan/T1024", - "functional_annotation/interproscan/T1024/T1024.gff3", - "functional_annotation/interproscan/T1024/T1024.json", - "functional_annotation/interproscan/T1024/T1024.tsv", - "functional_annotation/interproscan/T1024/T1024.xml", - "functional_annotation/interproscan/T1026", - "functional_annotation/interproscan/T1026/T1026.gff3", - "functional_annotation/interproscan/T1026/T1026.json", - "functional_annotation/interproscan/T1026/T1026.tsv", - "functional_annotation/interproscan/T1026/T1026.xml", - "functional_annotation/interproscan/l_arginase", - "functional_annotation/interproscan/l_arginase/l_arginase.gff3", - "functional_annotation/interproscan/l_arginase/l_arginase.json", - "functional_annotation/interproscan/l_arginase/l_arginase.tsv", - "functional_annotation/interproscan/l_arginase/l_arginase.xml", + "functional_annotation/interproscan/T1024_batch0", + "functional_annotation/interproscan/T1024_batch0/T1024_batch0.gff3", + "functional_annotation/interproscan/T1024_batch0/T1024_batch0.json", + "functional_annotation/interproscan/T1024_batch0/T1024_batch0.tsv", + "functional_annotation/interproscan/T1024_batch0/T1024_batch0.xml", + "functional_annotation/interproscan/T1026_batch0", + "functional_annotation/interproscan/T1026_batch0/T1026_batch0.gff3", + "functional_annotation/interproscan/T1026_batch0/T1026_batch0.json", + "functional_annotation/interproscan/T1026_batch0/T1026_batch0.tsv", + "functional_annotation/interproscan/T1026_batch0/T1026_batch0.xml", + "functional_annotation/interproscan/l_arginase_batch0", + "functional_annotation/interproscan/l_arginase_batch0/l_arginase_batch0.gff3", + "functional_annotation/interproscan/l_arginase_batch0/l_arginase_batch0.json", + "functional_annotation/interproscan/l_arginase_batch0/l_arginase_batch0.tsv", + "functional_annotation/interproscan/l_arginase_batch0/l_arginase_batch0.xml", "multiqc", "multiqc/multiqc_data", "multiqc/multiqc_data/llms-full.txt", @@ -182,12 +182,18 @@ "TIGRFAMs_15.0_HMM.LIB:md5,64f2b2c9e834b47b17d91bb9a6a0067e", "TIGRFAMs_HMM.LIB:md5,543da3f4b65eed9ec393986c6c6ff0ba", "interproscan_test.tar.gz:md5,cde88c0cd841c84dc1203e64854c762b", - "T1024.json:md5,0288f7551a14faedc409dd374b3e073e", - "T1024.xml:md5,63a3db0eb0e1f76403411602c23b721e", - "T1026.json:md5,5c2a40474b1cfb50cd043fe0be5e5d52", - "T1026.xml:md5,335552ce1703548565212a1d54681d75", - "l_arginase.json:md5,e0d127dd8a952cbd798999851d1338e6", - "l_arginase.xml:md5,7248992d9c1618cf7baa7515ae79ce32", + "T1024_batch0.gff3:md5,14701bc0093dced3ce9747ae5fc154f7", + "T1024_batch0.json:md5,0288f7551a14faedc409dd374b3e073e", + "T1024_batch0.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "T1024_batch0.xml:md5,63a3db0eb0e1f76403411602c23b721e", + "T1026_batch0.gff3:md5,14701bc0093dced3ce9747ae5fc154f7", + "T1026_batch0.json:md5,5c2a40474b1cfb50cd043fe0be5e5d52", + "T1026_batch0.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "T1026_batch0.xml:md5,335552ce1703548565212a1d54681d75", + "l_arginase_batch0.gff3:md5,26d1fe410d320fea4cb4e635359c9535", + "l_arginase_batch0.json:md5,e0d127dd8a952cbd798999851d1338e6", + "l_arginase_batch0.tsv:md5,e72024f44571a780abb597b4fe0a6a82", + "l_arginase_batch0.xml:md5,7248992d9c1618cf7baa7515ae79ce32", "multiqc_T1024_after.txt:md5,f2a552d4750ff8360941b10cec141499", "multiqc_T1024_before.txt:md5,f2a552d4750ff8360941b10cec141499", "multiqc_T1026_after.txt:md5,aabd4e58ed67d366fd04592ca09dbc9b", @@ -224,4 +230,4 @@ }, "timestamp": "2026-02-04T12:43:32.273407057" } -} \ No newline at end of file +} From c70cd5849c80b5b1b7477a451a03ad108eb4191a Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 27 Jul 2026 17:03:43 -0700 Subject: [PATCH 4/4] Fix functional_annotation test missing interproscan_batch_size input The FUNCTIONAL_ANNOTATION subworkflow test (newly added by the dev merge) only passed 4 inputs, but the subworkflow now takes a 5th param (interproscan_batch_size) for batch processing. Also refresh default.nf.test.snap for the two l_arginase InterProScan checksums that were stale relative to the merged pipeline state. --- .../local/functional_annotation/tests/main.nf.test | 2 ++ tests/default.nf.test.snap | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/subworkflows/local/functional_annotation/tests/main.nf.test b/subworkflows/local/functional_annotation/tests/main.nf.test index 8ea5d3d..3d433ab 100644 --- a/subworkflows/local/functional_annotation/tests/main.nf.test +++ b/subworkflows/local/functional_annotation/tests/main.nf.test @@ -21,6 +21,7 @@ nextflow_workflow { input[1] = false input[2] = params.pipelines_testdata_base_path + '/testdata/interproscan/interproscan_test.tar.gz' input[3] = [] + input[4] = 1000 """ } } @@ -49,6 +50,7 @@ nextflow_workflow { input[1] = true input[2] = [] input[3] = [] + input[4] = 1000 """ } } diff --git a/tests/default.nf.test.snap b/tests/default.nf.test.snap index 3696c61..f2e3e44 100644 --- a/tests/default.nf.test.snap +++ b/tests/default.nf.test.snap @@ -214,9 +214,9 @@ "T1026_batch0.json:md5,5c2a40474b1cfb50cd043fe0be5e5d52", "T1026_batch0.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", "T1026_batch0.xml:md5,335552ce1703548565212a1d54681d75", - "l_arginase_batch0.gff3:md5,26d1fe410d320fea4cb4e635359c9535", + "l_arginase_batch0.gff3:md5,3362ac4dde1351d84ef66d81369297f1", "l_arginase_batch0.json:md5,e0d127dd8a952cbd798999851d1338e6", - "l_arginase_batch0.tsv:md5,e72024f44571a780abb597b4fe0a6a82", + "l_arginase_batch0.tsv:md5,3007fcbf69e07173deb698ad465f5545", "l_arginase_batch0.xml:md5,7248992d9c1618cf7baa7515ae79ce32", "multiqc_T1024_after.txt:md5,f2a552d4750ff8360941b10cec141499", "multiqc_T1024_before.txt:md5,f2a552d4750ff8360941b10cec141499", @@ -248,10 +248,10 @@ "GI|225038609|EFDID|719595|FULL.ss2:md5,e7d8eaa84d46a6a714ffe00d7f21cdfb" ] ], - "timestamp": "2026-05-05T11:10:57.929189129", "meta": { - "nf-test": "0.9.5", - "nextflow": "26.04.0" - } + "nf-test": "0.9.2", + "nextflow": "25.10.4" + }, + "timestamp": "2026-07-27T17:01:25.693103" } -} +} \ No newline at end of file