Tims connection fix (#890)

* Divide tims Frames RT by 60 to convert to minutes, added one over K0 field to SpectrumMatchFromTsvHeader

* fixed dynamic connection issue in timsTofFileReader

* Reverted local-only changes

* Added test coverage for timsTOF error handling

* Changed method name in TimsDataScan, added project level ExcludeFromCodeCov tag to Test project

---------

Co-authored-by: trishorts <mshort@chem.wisc.edu>

Author: Alexander-Sol

mzLib/MassSpectrometry/MsDataFile.cs

@@ -30,9 +30,8 @@ public abstract class MsDataFile : IEnumerable<MsDataScan>
     {
         public MsDataScan[] Scans { get; protected set; }
         public SourceFile SourceFile { get; set; }
-        public int NumSpectra => Scans.Length;
+        public int NumSpectra => Scans?.Length ?? 0;
         public string FilePath { get; }
-
         protected MsDataFile(int numSpectra, SourceFile sourceFile)
         {
             Scans = new MsDataScan[numSpectra];

mzLib/Readers/ExternalResults/IndividualResultRecords/MsFraggerPsm.cs

@@ -212,8 +212,6 @@ public class MsFraggerPsm : IQuantifiableRecord
 
         [Ignore] private List<(string, string, string)> _proteinGroupInfos;
 
-        
-
         [Ignore] public int ChargeState => Charge;
 
         // decoy reading isn't currently supported for MsFragger psms, this will be revisited later

mzLib/Readers/timsTOF/FrameProxy.cs

@@ -1,4 +1,5 @@
 using MassSpectrometry;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 namespace Readers
@@ -32,7 +33,10 @@ internal FrameProxyFactory(FrameTable table, UInt64 fileHandle, Object fileLock,
             InitializeLookupTables(fileHandle);
         }   
 
-        internal FrameProxy GetFrameProxy(long frameId)
+        /// <summary>
+        /// This class is marked virtual for testing purposes only.
+        /// </summary>
+        internal virtual FrameProxy GetFrameProxy(long frameId)
         {
             return new FrameProxy(FileHandle, frameId, FramesTable.NumScans[frameId - 1], FileLock, Converter);
         }
@@ -155,6 +159,24 @@ internal FrameProxy(UInt64 fileHandle, long frameId, int numScans, Object fileLo
             _scanOffsets = PartialSum(_rawData, 0, numScans);
         }
 
+
+        /// <summary>
+        /// Sometimes, with corrupted data, the _scanOffsets array will specify a scan range that is 
+        /// greater than the legnth of the _rawData array or is negative. This method checks if the frame is valid
+        /// </summary>
+        /// <returns></returns>
+        internal bool IsFrameValid()
+        {
+            // All offsets should be non-negative and smaller than tge _rawData length
+            for (int i = 0; i < _scanOffsets.Length - 1; i++)
+            {
+                if (_scanOffsets[i] < 0 || _scanOffsets[i] > _rawData.Length)
+                    return false;
+            }
+
+            return true;
+        }
+
         /// <summary>
         /// Gets the intensities for the specified scan.
         /// </summary>
@@ -259,7 +281,12 @@ private void ThrowIfInvalidScanNumber(int zeroIndexedScanNumber)
         private Range GetScanRange(int zeroIndexedScanNumber, int offset)
         {
             int start = NumberOfScans + 2*_scanOffsets[zeroIndexedScanNumber] + offset;
-            return new Range(start, start + (int)_rawData[zeroIndexedScanNumber]);
+            int end = Math.Min(_rawData.Length, start + (int)_rawData[zeroIndexedScanNumber]);
+            if (start >= _rawData.Length)
+            {
+                throw new ArgumentException("Scan data exceeds raw data array length. This indicates that the .tdf_bin file is corrupted");
+            }
+            return new Range(start, end);
         }
 
         /// <summary>

mzLib/Readers/timsTOF/TimsDataScan.cs

@@ -66,9 +66,10 @@ public TimsDataScan(MzSpectrum massSpectrum,
             ComponentSpectraTotalPeaks = 0;
         }
 
-        internal void AverageComponentSpectra(FrameProxyFactory proxyFactory, FilteringParams filteringParams = null)
+        internal void SumComponentSpectra(FrameProxyFactory proxyFactory, FilteringParams filteringParams = null)
         {
             MassSpectrum = TofSpectraMerger.MergeArraysToMs2Spectrum(mzArrays, intensityArrays, filteringParams);
+            if (MassSpectrum == null) return;
             TotalIonCurrent = MassSpectrum.SumOfAllY;
             mzArrays.Clear();
             intensityArrays.Clear();

mzLib/Readers/timsTOF/TimsTofFileReader.cs

@@ -17,17 +17,21 @@
 using System.Diagnostics;
 using System.Security.Permissions;
 using System.ComponentModel;
+using System.Runtime.CompilerServices;
 
 namespace Readers
-{ 
+{
     public class TimsTofFileReader : MsDataFile, IDisposable
     {
         // timsTOF instruments collect frames, packets of ions collected by the tims, then analyzed 
         // over multiple scans with each scan corresponding to the same retention time but different
         // ion mobility valuess. When reading the file, multiple scans from the same frame are collapsed into 
         // a single spectrum
 
-        public TimsTofFileReader(string filePath) : base (filePath) { }
+        public TimsTofFileReader(string filePath) : base (filePath) 
+        {
+            FaultyFrameIds = new();
+        }
 
         private UInt64? _fileHandle;
         private Object _fileLock;
@@ -36,7 +40,12 @@ public TimsTofFileReader(string filePath) : base (filePath) { }
         public int NumberOfFrames { get; private set; }
         public List<long> Ms1FrameIds { get; private set; }
         internal FrameProxyFactory FrameProxyFactory { get; private set; }
-        
+
+        internal List<long> FaultyFrameIds { get; }
+
+        public string? Warnings => FaultyFrameIds.Count > 0 ? 
+            "The following frames were not read correctly: " + String.Join(", ", FaultyFrameIds) : null;
+
         // I don't know what the default scan range is, and at this point I'm too afraid to ask...
         private MzRange? _scanWindow;
         public MzRange ScanWindow => _scanWindow ??= new MzRange(20, 2000);
@@ -49,6 +58,9 @@ public override void InitiateDynamicConnection()
                 throw new FileNotFoundException("Data file is missing .tdf and/or .tdf_bin file");
             }
 
+            if (Scans.IsNotNullOrEmpty() && Scans.All(s => s != null)) // If all scans have been loaded, then don't reload
+                return;
+
             OpenSqlConnection();
 
             if(_fileHandle != null) tims_close((UInt64)_fileHandle);
@@ -57,6 +69,8 @@ public override void InitiateDynamicConnection()
 
             CountFrames();
             BuildProxyFactory();
+            CountMS1Frames();
+            CountPrecursors();
         }
 
         internal void OpenSqlConnection()
@@ -89,6 +103,7 @@ public override void CloseDynamicConnection()
         {
             if (_sqlConnection?.State == ConnectionState.Open) _sqlConnection.Close();
             _sqlConnection?.Dispose();
+            _sqlConnection = null;
             if (_fileHandle != null)
             {
                 tims_close((UInt64)_fileHandle);
@@ -232,9 +247,6 @@ public override MsDataFile LoadAllStaticData(FilteringParams filteringParams = n
         {
             InitiateDynamicConnection();
 
-            CountMS1Frames();
-            CountPrecursors();
-            
             _maxThreads = maxThreads;
             Ms1ScansNoPrecursorsBag = new();
             Parallel.ForEach(
@@ -315,6 +327,11 @@ internal void AssignOneBasedPrecursorsToPasefScans()
         internal void BuildAllScans(long frameId, FilteringParams filteringParams)
         {
             FrameProxy frame = FrameProxyFactory.GetFrameProxy(frameId);
+            if (frame == null || !frame.IsFrameValid())
+            {
+                FaultyFrameIds.Add(frameId);
+                return; // If the frame is null, then we can't build any scans for it
+            }
             var records = GetMs1Records(frameId);
             foreach(Ms1Record record in records)
             {
@@ -453,40 +470,7 @@ internal List<TimsDataScan> BuildPasefScanFromPrecursor(IEnumerable<int> precurs
                 pasefScans.Add(dataScan);
             }
 
-            // Grab all fragmentation spectra for each precursor
-            // Each TimsDataScan in pasefScans corresponds to one precursor.
-            // A precursor can be isolated and fragmented in multiple pasef frames
-            // Here, we iterate through each frame, averaging the scans that correspond to each precursor
-            foreach (long frameId in allFrames)
-            {
-                FrameProxy frame = FrameProxyFactory.GetFrameProxy(frameId);
-                //Iterate through all the datascans created above with this frame
-                foreach (var scan in pasefScans)
-                {
-                    if (scan.FrameIds.Contains(frameId))
-                    {
-                        List<uint[]> indexArrays = new();
-                        List<int[]> intensityArrays = new();
-                        for (int mobilityScanIdx = scan.ScanNumberStart; mobilityScanIdx < scan.ScanNumberEnd; mobilityScanIdx++)
-                        {
-                            indexArrays.Add(frame.GetScanIndices(mobilityScanIdx-1));
-                            intensityArrays.Add(frame.GetScanIntensities(mobilityScanIdx-1));
-                        }
-                        // Perform frame level averaging, where all scans from one frame associated with a given precursor are merged and centroided
-                        // Need to convert indexArrays to one uint[] and intensityArrays to one int[]
-                        (double[] Mzs, int[] Intensities) summedArrays = TofSpectraMerger.MergeArraysToMzArray(indexArrays, intensityArrays, FrameProxyFactory);
-                        scan.AddComponentArrays(summedArrays.Mzs, summedArrays.Intensities);
-                    }
-                }
-            }
-
-            // Now, we average the fragmentation spectra (each spectra originating in a different frame)
-            // to yield one spectrum per precursor
-            foreach (TimsDataScan scan in pasefScans)
-            {
-                scan.AverageComponentSpectra(FrameProxyFactory, filteringParams);
-            }
-
+            PopulateSpectraForPasefScans(pasefScans, allFrames, filteringParams);
             return pasefScans;
         }
 
@@ -530,6 +514,54 @@ internal IEnumerable<PasefRecord> GetPasefRecords(IEnumerable<int> precursorIds)
             }
         }
 
+        /// <summary>
+        /// Grab all fragmentation spectra for each precursor
+        /// Each TimsDataScan in pasefScansWithNullSpectra corresponds to one precursor.
+        /// A precursor can be isolated and fragmented in multiple pasef frames
+        /// Here, we iterate through each frame, averaging the scans that correspond to each precursor
+        /// </summary>
+        /// <param name="pasefScansWithNullSpectra">List of timsDataScans with metadata but no MzSpectrum</param>
+        /// <param name="relevantFrameIds">Frames that contains scans to average for the given pasefScans</param>
+        /// <param name="filteringParams">Filtering params that specify MS2 spectrum filtering options</param>
+        internal void PopulateSpectraForPasefScans(List<TimsDataScan> pasefScansWithNullSpectra, IEnumerable<long> relevantFrameIds, FilteringParams filteringParams)
+        {
+            foreach (long frameId in relevantFrameIds)
+            {
+                FrameProxy frame = FrameProxyFactory.GetFrameProxy(frameId);
+                if (frame == null || !frame.IsFrameValid())
+                {
+                    FaultyFrameIds.Add(frameId);
+                    continue; // If the frame is null, then we can't build any scans for it
+                }
+                //Iterate through all the datascans created above with this frame
+                foreach (var scan in pasefScansWithNullSpectra)
+                {
+                    if (scan.FrameIds.Contains(frameId))
+                    {
+                        List<uint[]> indexArrays = new();
+                        List<int[]> intensityArrays = new();
+                        for (int mobilityScanIdx = scan.ScanNumberStart; mobilityScanIdx < scan.ScanNumberEnd; mobilityScanIdx++)
+                        {
+                            indexArrays.Add(frame.GetScanIndices(mobilityScanIdx - 1));
+                            intensityArrays.Add(frame.GetScanIntensities(mobilityScanIdx - 1));
+                        }
+                        // Perform frame level averaging, where all scans from one frame associated with a given precursor are merged and centroided
+                        // Need to convert indexArrays to one uint[] and intensityArrays to one int[]
+                        (double[] Mzs, int[] Intensities) summedArrays = TofSpectraMerger.MergeArraysToMzArray(indexArrays, intensityArrays, FrameProxyFactory);
+                        scan.AddComponentArrays(summedArrays.Mzs, summedArrays.Intensities);
+                    }
+                }
+            }
+
+            // Now, we average the fragmentation spectra (each spectra originating in a different frame)
+            // to yield one spectrum per precursor
+            foreach (TimsDataScan scan in pasefScansWithNullSpectra)
+            {
+                scan.SumComponentSpectra(FrameProxyFactory, filteringParams);
+            }
+            pasefScansWithNullSpectra.RemoveAll(scan => scan.MassSpectrum == null || scan.MassSpectrum.Size < 1);
+        }
+
         private const string nativeIdFormat = "Frame ID + scan number range format";
         private const string massSpecFileFormat = ".D format";
         public override SourceFile GetSourceFile()