Quantify a design with a numbering gap, with a warning in the log and results.txt, instead of refusing it

MetaMorpheus/CMD/CommandLineSettings.cs

@@ -49,7 +49,7 @@ public class CommandLineSettings
         [Option("auditData", HelpText = "[Optional] Folder of downloaded data files to check the SDRF's comment[data file] entries against, reported as found or missing. Only meaningful with --auditSdrf.")]
         public string AuditDataDirectory { get; set; }
 
-        [Option("sdrfDesign", HelpText = "[Optional] Path to a label-free SDRF file to turn into an ExperimentalDesign.tsv, which is written beside the first spectra file given with -s, where a run looks for it. Every spectra file must be named exactly by one SDRF row; rows for other files are dropped and reported. Biological replicates are renumbered within each condition and the renumbering is printed; fractions are copied as they are. If the SDRF describes a design MetaMorpheus would reject, nothing is written and every reason is printed. An existing design file is never overwritten. Runs on its own: give -s, but no task or database. -o is ignored, with a warning, because a run reads the design only from beside its first spectra file. Exits 0 when the design was written, 2 for a settings error, 4 when the SDRF could not be read and 5 when the design was refused or a design file is already there.")]
+        [Option("sdrfDesign", HelpText = "[Optional] Path to a label-free SDRF file to turn into an ExperimentalDesign.tsv, which is written beside the first spectra file given with -s, where a run looks for it. Every spectra file must be named exactly by one SDRF row; rows for other files are dropped and reported. Biological replicates and fractions are copied as the SDRF numbers them; a replicate number is added only where the SDRF gives none, and every added number is printed, as is any gap in a condition's replicate numbers. If the SDRF describes a design MetaMorpheus would reject, nothing is written and every reason is printed. An existing design file is never overwritten. Runs on its own: give -s, but no task or database. -o is ignored, with a warning, because a run reads the design only from beside its first spectra file. Exits 0 when the design was written, 2 for a settings error, 4 when the SDRF could not be read and 5 when the design was refused or a design file is already there.")]
         public string SdrfDesign { get; set; }
 
         [Option("sdrfCondition", HelpText = "[Optional] The SDRF factor value columns the condition is built from, in order; space-delimited. A bare name such as 'genotype' means 'factor value[genotype]'; the 'factor value[' prefix may be given in any case. Values are joined with '_'. Needed when the SDRF has more than one factor value column. Only meaningful with --sdrfDesign.")]

MetaMorpheus/CMD/Program.cs

@@ -422,21 +422,34 @@ public static int ResolveExperimentalDesign(
 
             string designPath = hasClassicDesign ? pathToExperDesign : pathToTmtDesign;
             List<string> errors;
+            var warnings = new List<string>();
 
             if (hasClassicDesign)
             {
-                ExperimentalDesign.ReadExperimentalDesign(designPath, startingRawFilenameList, out errors);
+                var design = ExperimentalDesign.ReadExperimentalDesign(designPath, startingRawFilenameList, out errors);
+                if (!errors.Any())
+                {
+                    warnings = ExperimentalDesign.GetWarningsInExperimentalDesign(design);
+                }
             }
             else
             {
-                TmtExperimentalDesign.Read(designPath, startingRawFilenameList, out errors);
+                var tmtFiles = TmtExperimentalDesign.Read(designPath, startingRawFilenameList, out errors);
+                if (!errors.Any())
+                {
+                    warnings = TmtExperimentalDesign.GetWarningsInDesign(tmtFiles);
+                }
             }
 
             if (!errors.Any())
             {
                 if (reportToConsole)
                 {
                     write("Read " + Path.GetFileName(designPath) + " successfully");
+                    foreach (string warning in warnings)
+                    {
+                        write(warning);
+                    }
                 }
 
                 return 0;
@@ -546,7 +559,7 @@ private static void MyTaskEngine_finishedWritingFileHandler(object sender, Singl
         /// Refuses rather than writes anything MetaMorpheus would reject, because an invalid design is
         /// worse than none: a run finding one skips quantification with only a warning. The report is
         /// printed whatever the verbosity, as for --auditSdrf: it is what was asked for, and it is the
-        /// only record of a biological-replicate renumbering.
+        /// only record of a biological replicate number added where the SDRF gives none.
         /// </summary>
         /// <returns>0 when the design was written; 4 when the SDRF could not be read; 5 when the design
         /// was refused or a design file is already there.</returns>

MetaMorpheus/EngineLayer/ExperimentalDesign.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using MassSpectrometry;
@@ -112,62 +113,76 @@ public static string WriteExperimentalDesignToFile(List<SpectraFileInfo> spectra
 
         /// <summary>
         /// Checks for errors in the experimental design. Will return null if there are no errors.
+        /// The one error is a duplicate: two files at the same condition, biorep, fraction and techrep.
+        /// A gap in biorep, fraction or techrep numbers is not an error. The numbers are kept as given
+        /// (a biorep number can name a subject across conditions, and a gap can be a lost sample) and
+        /// reported by GetWarningsInExperimentalDesign.
         /// </summary>
         public static string GetErrorsInExperimentalDesign(List<SpectraFileInfo> spectraFileInfos)
         {
-            // check for correct iteration of integer values and duplicates
-            var conditions = spectraFileInfos.GroupBy(p => p.Condition);
-
-            foreach (var condition in conditions)
+            var duplicate = spectraFileInfos
+                .GroupBy(p => (p.Condition, p.BiologicalReplicate, p.Fraction, p.TechnicalReplicate))
+                .Where(g => g.Count() > 1)
+                .Select(g => g.Key)
+                .OrderBy(k => k.Condition, StringComparer.Ordinal).ThenBy(k => k.BiologicalReplicate).ThenBy(k => k.Fraction).ThenBy(k => k.TechnicalReplicate)
+                .FirstOrDefault();
+
+            if (duplicate != default)
             {
-                var temp = condition.OrderBy(p => p.BiologicalReplicate).ThenBy(p => p.Fraction).ThenBy(p => p.TechnicalReplicate);
-                int numB = temp.Max(p => p.BiologicalReplicate + 1);
+                return "Duplicates are not allowed:\n" +
+                    "Condition \"" + duplicate.Condition + "\" biorep " + (duplicate.BiologicalReplicate + 1) +
+                    " fraction " + (duplicate.Fraction + 1) + " techrep " + (duplicate.TechnicalReplicate + 1);
+            }
 
-                // check bioreps are in order
-                for (int b = 0; b < numB; b++)
-                {
-                    var biorepFiles = temp.Where(p => p.BiologicalReplicate == b);
+            return null;
+        }
 
-                    if (!biorepFiles.Any())
-                    {
-                        return "Condition \"" + condition.Key + "\" biorep " + (b + 1) + " is missing!";
-                    }
+        /// <summary>
+        /// One warning for each place the numbering has a gap, naming the missing numbers: a condition's
+        /// bioreps, a biorep's fractions, and a fraction's techreps, each expected to run 1..N. Such a design
+        /// is quantified as numbered; the warning is there because a missing number may be a sample or
+        /// file that was lost or not searched. Empty when there is nothing to say.
+        /// </summary>
+        public static List<string> GetWarningsInExperimentalDesign(List<SpectraFileInfo> spectraFileInfos)
+        {
+            var warnings = new List<string>();
+
+            foreach (var condition in spectraFileInfos.GroupBy(p => p.Condition).OrderBy(c => c.Key, StringComparer.Ordinal))
+            {
+                string conditionLabel = "Condition \"" + condition.Key + "\"";
+                AddGapWarning(warnings, conditionLabel, "biological replicate", condition.Select(p => p.BiologicalReplicate));
 
-                    // check fractions are in order
-                    int numF = biorepFiles.Max(p => p.Fraction + 1);
+                foreach (var biorep in condition.GroupBy(p => p.BiologicalReplicate).OrderBy(b => b.Key))
+                {
+                    string biorepLabel = conditionLabel + " biorep " + (biorep.Key + 1);
+                    AddGapWarning(warnings, biorepLabel, "fraction", biorep.Select(p => p.Fraction));
 
-                    for (int f = 0; f < numF; f++)
+                    foreach (var fraction in biorep.GroupBy(p => p.Fraction).OrderBy(f => f.Key))
                     {
-                        var fractionFiles = biorepFiles.Where(p => p.Fraction == f);
-
-                        if (!fractionFiles.Any())
-                        {
-                            return "Condition \"" + condition.Key + "\" biorep " + (b + 1) + " fraction " + (f + 1) + " is missing!";
-                        }
-
-                        // check techreps are in order
-                        int numT = fractionFiles.Max(p => p.TechnicalReplicate + 1);
-
-                        for (int t = 0; t < numT; t++)
-                        {
-                            var techrepFiles = fractionFiles.Where(p => p.TechnicalReplicate == t);
-
-                            if (!techrepFiles.Any())
-                            {
-                                return "Condition \"" + condition.Key + "\" biorep " + (b + 1) + " fraction " + (f + 1) + " techrep " + (t + 1) + " is missing!";
-                            }
-
-                            if (techrepFiles.Count() > 1)
-                            {
-                                return "Duplicates are not allowed:\n" +
-                                    "Condition \"" + condition.Key + "\" biorep " + (b + 1) + " fraction " + (f + 1) + " techrep " + (t + 1);
-                            }
-                        }
+                        AddGapWarning(warnings, biorepLabel + " fraction " + (fraction.Key + 1), "technical replicate", fraction.Select(p => p.TechnicalReplicate));
                     }
                 }
             }
 
-            return null;
+            return warnings;
+        }
+
+        /// <summary>
+        /// Adds a warning when the given zero-based numbers, read one-based, are not 1..N. Worded as mzLib's
+        /// SdrfLabelFreeDesign notes the same gap ("Condition 'A' biorep 1: fractions 1, 3, kept as the SDRF
+        /// numbers them."), so the SDRF report and the run say one thing.
+        /// </summary>
+        private static void AddGapWarning(List<string> warnings, string owner, string level, IEnumerable<int> zeroBasedNumbers)
+        {
+            var present = zeroBasedNumbers.Select(n => n + 1).Distinct().OrderBy(n => n).ToList();
+            var missing = Enumerable.Range(1, Math.Max(0, present.Max())).Except(present).ToList();
+
+            if (missing.Any())
+            {
+                warnings.Add(owner + ": " + level + "s " + string.Join(", ", present) + ", quantified as numbered; " +
+                    level + (missing.Count == 1 ? " " : "s ") + string.Join(", ", missing) + (missing.Count == 1 ? " is" : " are") + " not in the design. " +
+                    "A missing number may be a sample or file that was lost or not searched.");
+            }
         }
     }
 }

MetaMorpheus/EngineLayer/TmtExperimentalDesign.cs

@@ -60,6 +60,88 @@ private static string ValidateUniqueSampleBioFracTech(IEnumerable<(string Plex,
             return string.Join(Environment.NewLine, msgs);
         }
 
+        /// <summary>
+        /// The per-file rules of a TMT design, scoped to a plex because a plex is its own labelling
+        /// experiment: every file has a plex, fractions and technical replicates are >= 1, and no plex
+        /// holds two files at one (fraction, technical replicate). Null when there is no error. A gap in
+        /// fraction or technical replicate numbers is not an error; see <see cref="GetWarningsInDesign"/>.
+        /// Lifted from the TMT design window (#2793), so the window and a test check the same rules.
+        /// </summary>
+        public static string GetErrorsInDesign(IEnumerable<TmtFileInfo> files)
+        {
+            var fileList = files.ToList();
+
+            // A file with no plex passes every per-plex check, and then does not appear in the written
+            // design at all, because the design is built by grouping on plex.
+            var unplexed = fileList.Where(f => string.IsNullOrWhiteSpace(f.Plex)).ToList();
+            if (unplexed.Any())
+            {
+                return "Every file must be assigned a Plex. Missing for: "
+                    + string.Join(", ", unplexed.Select(f => Path.GetFileName(f.FullFilePathWithExtension)));
+            }
+
+            if (fileList.Any(f => f.TechnicalReplicate < 1))
+                return "Technical Replicate values must be >= 1.";
+
+            if (fileList.Any(f => f.Fraction < 1))
+                return "Fraction values must be >= 1.";
+
+            // Scoped to one plex: two files in different plexes both at (Fraction 1, Technical Replicate 1)
+            // is the correct design for a two-plex run with one fraction and one technical replicate.
+            foreach (var plexGroup in fileList.GroupBy(f => f.Plex.Trim(), StringComparer.OrdinalIgnoreCase))
+            {
+                var duplicatePair = plexGroup.GroupBy(f => (f.Fraction, f.TechnicalReplicate))
+                                             .FirstOrDefault(g => g.Count() > 1);
+                if (duplicatePair != null)
+                    return $"Plex {plexGroup.Key}: duplicate Fraction/Technical Replicate combination: Fraction {duplicatePair.Key.Fraction}, Technical Replicate {duplicatePair.Key.TechnicalReplicate}.";
+            }
+
+            return null;
+        }
+
+        /// <summary>
+        /// One warning for each place a plex's numbering has a gap, naming the missing numbers: its
+        /// fractions, and each fraction's technical replicates, each expected to run 1..N. Such a design
+        /// is quantified as numbered; the warning is there because a missing number may be a file that
+        /// was lost or not searched. Empty when there is nothing to say.
+        /// </summary>
+        public static List<string> GetWarningsInDesign(IEnumerable<TmtFileInfo> files)
+        {
+            var warnings = new List<string>();
+
+            foreach (var plexGroup in files.Where(f => !string.IsNullOrWhiteSpace(f.Plex))
+                         .GroupBy(f => f.Plex.Trim(), StringComparer.OrdinalIgnoreCase)
+                         .OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase))
+            {
+                string plexLabel = $"Plex {plexGroup.Key}";
+                AddGapWarning(warnings, plexLabel, "fraction", plexGroup.Select(f => f.Fraction));
+
+                foreach (var fraction in plexGroup.GroupBy(f => f.Fraction).OrderBy(g => g.Key))
+                {
+                    AddGapWarning(warnings, $"{plexLabel} fraction {fraction.Key}", "technical replicate",
+                        fraction.Select(f => f.TechnicalReplicate));
+                }
+            }
+
+            return warnings;
+        }
+
+        /// <summary>
+        /// Adds a warning when the given one-based numbers are not 1..N.
+        /// </summary>
+        private static void AddGapWarning(List<string> warnings, string owner, string level, IEnumerable<int> oneBasedNumbers)
+        {
+            var present = oneBasedNumbers.Distinct().OrderBy(n => n).ToList();
+            var missing = Enumerable.Range(1, Math.Max(0, present.Max())).Except(present).ToList();
+
+            if (missing.Any())
+            {
+                warnings.Add($"{owner}: {level}s {string.Join(", ", present)}, quantified as numbered; " +
+                    $"{level}{(missing.Count == 1 ? "" : "s")} {string.Join(", ", missing)} {(missing.Count == 1 ? "is" : "are")} not in the design. " +
+                    "A missing number may be a file that was lost or not searched.");
+            }
+        }
+
         // RETURN: files where each file carries its plex annotations
         /// <param name="fullFilePathsWithExtension">
         /// The files this run actually searched, used to ignore design rows naming anything else.

MetaMorpheus/GUI/Views/ExperimentalDesignWindow.xaml.cs

@@ -81,6 +81,13 @@ private void BtnSaveQuant_Click(object sender, RoutedEventArgs e)
                     .Select(p => new SpectraFileInfo(p.FullFilePathWithExtension, p.Condition, int.Parse(p.Biorep) - 1, int.Parse(p.Techrep) - 1, int.Parse(p.Fraction) - 1))
                     .ToList();
                 ExperimentalDesign.WriteExperimentalDesignToFile(fileInfos);
+
+                // saved as numbered; said, because a skipped biorep number is often a typo
+                var warnings = ExperimentalDesign.GetWarningsInExperimentalDesign(fileInfos);
+                if (warnings.Any())
+                {
+                    MessageBox.Show("Experimental design saved.\n\n" + string.Join("\n\n", warnings));
+                }
             }
             catch (Exception ex)
             {