Clusterinfo (#121)

* add parent mass from cluster summary

* add parent mass to test data

* replace ms2_match with library_match

* remove adduct, add parent mass and RT

* change parent mz into bool param

* add mz RT, remove adduct

* specify index for library matches

* parent mz to bool, ms2_matches to library_matches

* fixing tests

* add clarifying comment

Author: anupriyatripathi

q2_qemistree/_hierarchy.py

@@ -59,7 +59,7 @@ def merge_feature_data(fdata: pd.DataFrame) -> pd.DataFrame:
 
 def make_hierarchy(csi_results: CSIDirFmt,
                    feature_tables: biom.Table,
-                   ms2_matches: pd.DataFrame = None,
+                   library_matches: pd.DataFrame = None,
                    qc_properties: bool = False,
                    metric: str = 'euclidean') -> (TreeNode, biom.Table,
                                                   pd.DataFrame):
@@ -75,7 +75,7 @@ def make_hierarchy(csi_results: CSIDirFmt,
         one or more CSI:FingerID output folder
     feature_table : biom.Table
         one or more feature tables with mass-spec feature intensity per sample
-    ms2_matches: pd.DataFrame
+    library_matches: pd.DataFrame
         one or more tables with MS/MS library match for mass-spec features
     qc_properties : bool, default False
         flag to filter molecular properties to keep only PUBCHEM fingerprints
@@ -104,20 +104,21 @@ def make_hierarchy(csi_results: CSIDirFmt,
     if len(feature_tables) != len(csi_results):
         raise ValueError("The feature tables and CSI results should have a "
                          "one-to-one correspondance.")
-    if ms2_matches and len(ms2_matches) != len(feature_tables):
+    if library_matches and len(library_matches) != len(feature_tables):
         raise ValueError("The MS2 match tables should have a one-to-one "
                          "correspondance with feature tables and CSI results.")
     for n, (feature_table, csi_result) in enumerate(zip(feature_tables,
                                                         csi_results)):
         if feature_table.is_empty():
             raise ValueError("Cannot have empty feature table")
-        if ms2_matches:
-            ms2_match = ms2_matches[n]
-            if 'Smiles' not in ms2_match.columns:
+        if library_matches:
+            library_match = library_matches[n]
+            if 'Smiles' not in library_match.columns:
                 raise ValueError("MS2 match tables must contain the "
                                  "column `Smiles`. Please check if you have "
                                  "the correct input file for this command.")
-            collated_fps, smiles = process_csi_results(csi_result, ms2_match,
+            collated_fps, smiles = process_csi_results(csi_result,
+                                                       library_match,
                                                        qc_properties,
                                                        metric=metric)
         else:

q2_qemistree/_match.py

@@ -66,16 +66,17 @@ def get_matched_tables(collated_fingerprints: pd.DataFrame,
         list_md5.append(md5)
     fps['label'] = list_md5
     filtered_table['label'] = list_md5
-    feature_data = pd.DataFrame(columns=['label', '#featureID',
-                                         'csi_smiles', 'ms2_smiles',
-                                         'ms2_library_match', 'ms2_adduct'])
+    feature_data = pd.DataFrame(columns=['label', '#featureID','csi_smiles',
+                                         'ms2_smiles','ms2_library_match',
+                                         'parent_mass', 'retention_time'])
     feature_data['label'] = list_md5
     feature_data['#featureID'] = allfps
     feature_data['csi_smiles'] = list(smiles.loc[allfps, 'csi_smiles'])
     feature_data['ms2_smiles'] = list(smiles.loc[allfps, 'ms2_smiles'])
     feature_data['ms2_library_match'] = list(
         smiles.loc[allfps, 'ms2_library_match'])
-    feature_data['ms2_adduct'] = list(smiles.loc[allfps, 'ms2_adduct'])
+    feature_data['parent_mass'] = list(smiles.loc[allfps, 'parent_mass'])
+    feature_data['retention_time'] = list(smiles.loc[allfps, 'retention_time'])
     feature_data.set_index('label', inplace=True)
     relabel_fps = fps.groupby('label').first()
     matched_table = filtered_table.groupby('label').sum()

q2_qemistree/_plot.py

@@ -80,15 +80,15 @@ def format_labels(feature_metadata, category, ms2_label, parent_mz):
                 label = feature_metadata.loc[idx, category]
 
             if parent_mz and label in missing_values:
-                label = feature_metadata.loc[idx, parent_mz]
+                label = feature_metadata.loc[idx, 'parent_mass']
 
             labels.append('%s\t%s' % (idx, label))
     else:
         for idx in feature_metadata.index:
             label = feature_metadata.loc[idx, category]
 
             if parent_mz and label in missing_values:
-                label = feature_metadata.loc[idx, parent_mz]
+                label = feature_metadata.loc[idx, 'parent_mass']
 
             labels.append('%s\t%s' % (idx, label))
 
@@ -125,8 +125,8 @@ def format_barplots(table: biom.Table):
 
 
 def plot(output_dir: str, tree: NewickFormat, feature_metadata: pd.DataFrame,
-         category: str = 'class', ms2_label: bool = True,
-         color_palette: str = 'Dark2', parent_mz: str = None,
+         category: str = 'class', ms2_label: bool = False,
+         color_palette: str = 'Dark2', parent_mz: bool = True,
          grouped_table: biom.Table = None) -> None:
     '''This function plots the phenetic tree in iTOL with clade colors,
     feature labels and relative abundance per sample group.
@@ -146,9 +146,9 @@ def plot(output_dir: str, tree: NewickFormat, feature_metadata: pd.DataFrame,
         The color palette to use for coloring tips
     ms2_label : bool, optional
         Whether to label the tips with the MS2 value
-    parent_mz : str, optional
-        If the feature is unclassified, label the tips using this
-        column's value
+    parent_mz : bool, optional
+        If the feature is unclassified, label the tips using parent mass of
+        compound
 
     Raises
     ------

q2_qemistree/_process_fingerprint.py

@@ -57,7 +57,7 @@ def collate_fingerprint(csi_result: CSIDirFmt, qc_properties: bool = False,
 
 
 def get_feature_smiles(csi_result: CSIDirFmt, collated_fps: pd.DataFrame,
-                       ms2_match: pd.DataFrame = None):
+                       library_match: pd.DataFrame = None):
     '''This function gets the SMILES of mass-spec features from
     CSI:FingerID and optionally, MS/MS library match results
     '''
@@ -70,26 +70,29 @@ def get_feature_smiles(csi_result: CSIDirFmt, collated_fps: pd.DataFrame,
     smiles['csi_smiles'] = csi_summary.loc[smiles.index, 'smiles'].str.strip()
     smiles['ms2_smiles'] = np.nan
     smiles['ms2_library_match'] = np.nan
-    smiles['ms2_adduct'] = np.nan
-    if ms2_match is not None:
-        ms2_match.index = ms2_match.index.astype(str)
-        ms2_ids = ms2_match.index.intersection(smiles.index)
-        smiles['ms2_smiles'] = ms2_match.loc[ms2_ids, 'Smiles'].str.strip()
-        smiles['ms2_library_match'] = ms2_match.loc[
-            ms2_ids, 'Compound_Name']
-        smiles['ms2_adduct'] = ms2_match.loc[ms2_ids, 'Adduct']
+    smiles['parent_mass'] = np.nan
+    smiles['retention_time'] = np.nan
+    if library_match is not None:
+        library_match.index = library_match.index.astype(str)
+        ms2_ids = library_match.index.intersection(smiles.index)
+        smiles['ms2_smiles'] = library_match.loc[ms2_ids, 'Smiles'].str.strip()
+        smiles['ms2_library_match'] = library_match.loc[ms2_ids, 'LibraryID']
+        smiles['parent_mass'] = library_match.loc[ms2_ids, 'parent mass']
+        smiles['retention_time'] = library_match.loc[ms2_ids, 'RTConsensus']
     smiles = smiles.fillna('missing').apply(
         lambda x: x.replace({' ': 'missing', '': 'missing'}))
     return smiles
 
 
-def process_csi_results(csi_result: CSIDirFmt, ms2_match: pd.DataFrame = None,
+def process_csi_results(csi_result: CSIDirFmt,
+                        library_match: pd.DataFrame = None,
                         qc_properties: bool = False,
                         metric: str = 'euclidean') -> (pd.DataFrame,
                                                        pd.DataFrame):
     '''This function parses CSI:FingerID result to generate tables
     of collated molecular fingerprints and SMILES for mass-spec features
     '''
     collated_fps = collate_fingerprint(csi_result, qc_properties, metric)
-    feature_smiles = get_feature_smiles(csi_result, collated_fps, ms2_match)
+    feature_smiles = get_feature_smiles(csi_result, collated_fps,
+                                        library_match)
     return collated_fps, feature_smiles

q2_qemistree/_transformer.py

@@ -8,8 +8,6 @@ def _read_dataframe(fh):
     # Using `dtype=object` and `set_index` to avoid type casting/inference
     # of any columns or the index.
     df = pd.read_csv(fh, sep='\t', header=0, dtype='str')
-    df.set_index(df.columns[0], drop=True, append=False, inplace=True)
-    df.index.name = 'id'
     return df
 
 # define a transformer from pd.DataFrame to -> TSVMolecules
@@ -25,6 +23,14 @@ def _1(data: pd.DataFrame) -> TSVMolecules:
 def _2(ff: TSVMolecules) -> pd.DataFrame:
     with ff.open() as fh:
         df = _read_dataframe(fh)
+        # Using 'cluster index' as index explicity for library matches
+        # since it may not be the first column
+        if 'cluster index' in df.columns:
+            df.set_index('cluster index', drop=True, append=False,
+                         inplace=True)
+        else:
+            df.set_index(df.columns[0], drop=True, append=False, inplace=True)
+        df.index.name = 'id'
         return df
 
 # define a transformer from TSVMolecules -> qiime2.Metadata

q2_qemistree/plugin_setup.py

@@ -162,21 +162,22 @@
     description='Build a phylogeny based on molecular substructures',
     inputs={'csi_results': List[CSIFolder],
             'feature_tables': List[FeatureTable[Frequency]],
-            'ms2_matches': List[FeatureData[Molecules]]},
+            'library_matches': List[FeatureData[Molecules]]},
     parameters={'qc_properties': Bool,
                 'metric': Str % Choices(['euclidean', 'jaccard'])},
     input_descriptions={'csi_results': 'one or more CSI:FingerID '
                                        'output folders',
                         'feature_tables': 'one or more feature tables with '
                                           'mass-spec feature intensity '
                                           'per sample',
-                        'ms2_matches': 'one or more tables with MS/MS library '
-                                       'match for mass-spec features'},
+                        'library_matches': 'one or more tables with MS/MS '
+                                           'library match for mass-spec '
+                                           'features'},
     parameter_descriptions={'qc_properties': 'filters molecular properties to '
                                              'retain PUBCHEM fingerprints',
                             'metric': 'metric for hierarchical clustering of '
-                                      'fingerprints. If the Jaccard metric is'
-                                      ' selected, molecular fingerprints are '
+                                      'fingerprints. If the Jaccard metric is '
+                                      'selected, molecular fingerprints are '
                                       'first binarized (probabilities above '
                                       '0.5 are True, and False otherwise).'},
     outputs=[('tree', Phylogeny[Rooted]),
@@ -245,7 +246,6 @@
             },
     parameters={
         'category': Str,
-        'parent_mz': Str,
         'color_palette': Str % Choices(['Pastel1', 'Pastel2', 'Paired',
                                         'Accent', 'Dark2', 'Set1', 'Set2',
                                         'Set3', 'tab10', 'tab20', 'tab20b',
@@ -255,6 +255,7 @@
                                         'BuPu', 'GnBu', 'PuBu', 'YlGnBu',
                                         'PuBuGn', 'BuGn', 'YlGn']),
         'ms2_label': Bool,
+        'parent_mz': Bool
                 },
     input_descriptions={'grouped_table': 'Feature table of samples '
                                          'grouped by categories. We recommend '
@@ -274,7 +275,7 @@
                          'tutorials/colors/colormaps.html',
         'ms2_label': 'Whether to label the tips with the MS2 value',
         'parent_mz': 'If the feature is unclassified, label the tips using '
-                     'this column\'s value'
+                     'this parent mass of the molecule'
     },
     citations=[citations['letunic2019itol']])
 

q2_qemistree/tests/data/feature_data_itol.txt

@@ -1,12 +1,12 @@
-label	#featureID	csi_smiles	ms2_smiles	table_number	important	not_so_important	structure_source	ms2_library_match	
-featurehash1	3	missing	missing	1	a	x	CSIFingerID	Spectral Match to Bleepbloop	
-featurehash2	7	CCCCCCCCCCCCCCCC(=O)OCC(COP(=O)([O-])OCC[N+](C)(C)C)O		1	a	x	MS2	Caffeine	
-featurehash3	2	missing	missing	1	b	x	MS2	Fakeiine	
-featurehash4	5	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	c	x	MS2	missing	
-featurehash5	6	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	d	x	MS2	missing	
-featurehash7	7	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	e	y	missing	Spectral Match to Glu-Val from NIST14	
-featurehash8	8	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	f	y	CSIFingerID	Spectral Match to Glu-Val from NIST14	
-featurehash10	10	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	g	y	CSIFingerID	missing	
-featurehash22	22	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	h	y	missing	missing	
-featurehash42	23	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	i	y	CSIFingerID	missing	
-featurehash43	24	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	j	y	missing	missing	
+label	#featureID	csi_smiles	ms2_smiles	table_number	important	not_so_important	structure_source	ms2_library_match	parent_mass	retention_time
+featurehash1	3	missing	missing	1	a	x	CSIFingerID	Spectral Match to Bleepbloop	193.2	4.1
+featurehash2	7	CCCCCCCCCCCCCCCC(=O)OCC(COP(=O)([O-])OCC[N+](C)(C)C)O		1	a	x	MS2	Caffeine	194.2	5.1
+featurehash3	2	missing	missing	1	b	x	MS2	Fakeiine	195.2	6.1
+featurehash4	5	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	c	x	MS2	missing	196.2	7.1
+featurehash5	6	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	d	x	MS2	missing	197.2	8.1
+featurehash7	7	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	e	y	missing	Spectral Match to Glu-Val from NIST14	198.2	9.1
+featurehash8	8	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	f	y	CSIFingerID	Spectral Match to Glu-Val from NIST14	199.2	10.1
+featurehash10	10	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	g	y	CSIFingerID	missing	200.2	11.1
+featurehash22	22	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	h	y	missing	missing	201.2	12.1
+featurehash42	23	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	i	y	CSIFingerID	missing	202.2	13.1
+featurehash43	24	CC(=NC(=O)CC(=NC(=O)C)OOC(=O)C)O		1	j	y	missing	missing	203.2	14.1

q2_qemistree/tests/data/features_smiles.txt

@@ -1,4 +1,4 @@
-#featureID	csi_smiles	ms2_smiles	ms2_library_match	ms2_adduct
-3				
-7	CC(=N)C1=CN(C)C=N1			
-2				
+#featureID	csi_smiles	ms2_smiles	ms2_library_match	parent_mass	retention_time
+3				100.5	1.5
+7	CC(=N)C1=CN(C)C=N1			200.5	2.5
+2				300.5	3.5

q2_qemistree/tests/data/ms2_match.txt

@@ -1,4 +1,4 @@
-#Scan#	Adduct	CAS_Number	Charge	Compound_Name	Compound_Source	Data_Collector	ExactMass	FileScanUniqueID	INCHI	INCHI_AUX	Instrument	IonMode	Ion_Source	LibMZ	LibraryName	LibraryQualityString	Library_Class	MQScore	MZErrorPPM	MassDiff	NumberHits	Organism	PI	Precursor_MZ	Pubmed_ID	RT_Query	SharedPeaks	Smiles	SpecCharge	SpecMZ	SpectrumFile	SpectrumID	TIC_Query	UpdateWorkflowName	tags
-3	M+H-H2O	5255174	1	Spectral Match to 3.beta.-Hydroxy-5-cholenoic acid from NIST14	Isolated	Data deposited by fevargas	0	spectra/specs_ms.mgf1	N/A	N/A	HCD	Positive	ESI	357.278	lib-00032.mgf	Bronze	3	0.863553	0.768752	0.000274658	1	GNPS-NIST14-MATCHES	Data from Christopher A. Lowry	357.278	N/A	0	14	N/A	1	357.278	spectra/specs_ms.mgf	CCMSLIB00003134721	173700	UPDATE-SINGLE-ANNOTATED-BRONZE	 
-4	149.1	628977	1	Spectral Match to Palmitic acid ethyl ester from NIST14	Isolated	Data deposited by mjmeehan	0	spectra/specs_ms.mgf1000	N/A	N/A	IT/ion trap	Positive	ESI	149.096	lib-00032.mgf	Bronze	3	0.662824	0	0	1	GNPS-NIST14-MATCHES	Data from Gabriel Haddad	149.096	N/A	0	3	N/A	1	149.096	spectra/specs_ms.mgf	CCMSLIB00003139076	3.50E+06	UPDATE-SINGLE-ANNOTATED-BRONZE	 
-2	M-H2O+H	n/a	1	"NCGC00380522-01_C20H30O4_1-Phenanthrenecarboxylic acid, 1,2,3,4,4a,4b,5,6,7,9,10,10a-dodecahydro-7-hydroxy-1,4a-dimethyl-7-(1-methylethyl)-9-oxo-"	isolated	lfnothias	334.214	spectra/specs_ms.mgf1031	"InChI=1S/C20H30O4/c1-12(2)20(24)9-6-14-13(11-20)15(21)10-16-18(14,3)7-5-8-19(16,4)17(22)23/h11-12,14,16,24H,5-10H2,1-4H3,(H,22,23)"	N/A	Maxis II HD Q-TOF Bruker	positive	LC-ESI	317.211	lib-00002.mgf	Gold	1	0.640657	0.673441	0.000213623	1	GNPS-NIH-NATURALPRODUCTSLIBRARY_ROUND2_POSITIVE	Jadhav/Dorrestein	317.211	n/a	0	8	CC(C)C\1(O)CCC2\C(=C1)C(=O)CC3C2(C)CCCC3(C)C(O)=O	1	317.211	spectra/specs_ms.mgf	CCMSLIB00000851506	664000	UPDATE-SINGLE-ANNOTATED-GOLD	 
+ATTRIBUTE_sample_type_group1	ATTRIBUTE_sample_type_group2	ATTRIBUTE_sample_type_group3	Annotated Adduct Features ID	Best Ion	Correlated Features Group ID	G1	G2	G3	G4	G5	G6	GNPSGROUP:achene	GNPSGROUP:algae	GNPSGROUP:animal	GNPSGROUP:complex	GNPSGROUP:dairy	GNPSGROUP:egg	GNPSGROUP:fleshy fruit	GNPSGROUP:fruit	GNPSGROUP:fungi	GNPSGROUP:game meat	GNPSGROUP:gastropod	GNPSGROUP:grain/grass	GNPSGROUP:honey	GNPSGROUP:insect	GNPSGROUP:invertebrate	GNPSGROUP:legume	GNPSGROUP:meat	GNPSGROUP:mineral	GNPSGROUP:not collected	GNPSGROUP:not provided	GNPSGROUP:nut	GNPSGROUP:plant	GNPSGROUP:poultry	GNPSGROUP:reptile	GNPSGROUP:sap	GNPSGROUP:seafood	GNPSGROUP:seaweed	GNPSGROUP:seed	GNPSGROUP:soap	GNPSGROUP:sugar	GNPSGROUP:supplement	GNPSGROUP:vegetable/herb	GNPSGROUP:water	GNPSLinkout_Cluster	GNPSLinkout_Network	INCHI	LibraryID	MQScore	MS2 Verification Comment	RTConsensus	RTMean	RTStdErr	Smiles	SpectrumID	SumPeakIntensity	UniqueFileSourcesCount	cluster index	componentindex	neutral M mass	number of spectra	parent mass	precursor charge	precursor mass	sum(precursor intensity)
+"fungi,algae,plant,animal"	"fruit,fungi,algae,animal"	"algae,fleshy fruit,seaweed,fungi,dairy"	 	 	 	0	0	0	0	0	0	0	350682.6827	371134.5664	0	1113403.699	0	108800.7569	108800.7569	44505.32383	0	0	0	0	0	0	0	0	0	0	0	0	108800.7569	0	0	0	0	350682.6827	0	0	0	0	0	0	"https://gnps.ucsd.edu/ProteoSAFe/result.jsp?task=096df6e409a14363ab59cba2fd530e80&view=view_all_clusters_withID&show=true#{""main.cluster index_lowerinput"":""1.0"",""main.cluster index_upperinput"":""1.0""}"	https://gnps.ucsd.edu/ProteoSAFe/result.jsp?view=network_displayer&componentindex=6&task=096df6e409a14363ab59cba2fd530e80&show=true	N/A	Spectral Match to 3.beta.-Hydroxy-5-cholenoic acid from NIST14	N/A	 	6.1091	6.1091	0	N/A	N/A	5504981.929	7	3	6	 	7	230.2476	1	230.2476	5504981.929
+"fungi,plant,animal"	"fruit,fungi,animal"	"fleshy fruit,fungi,honey,dairy"	 	 	 	0	0	0	0	0	0	0	0	605688.934	0	1739539.412	0	518945.9818	518945.9818	128470.8033	0	0	0	77527.38967	0	0	0	0	0	0	0	0	518945.9818	0	0	0	0	0	0	0	0	0	0	0	"https://gnps.ucsd.edu/ProteoSAFe/result.jsp?task=096df6e409a14363ab59cba2fd530e80&view=view_all_clusters_withID&show=true#{""main.cluster index_lowerinput"":""2.0"",""main.cluster index_upperinput"":""2.0""}"	This Node is a Singleton	N/A	Spectral Match to Palmitic acid ethyl ester from NIST14	N/A	 	0.2616	0.2616	0	N/A	N/A	10507126.65	15	4	-1	 	15	381.0796	1	381.0796	10507126.65
+animal	animal	"honey,dairy"	 	 	 	0	0	0	0	0	0	0	0	736410.3824	0	2192431.97	0	0	0	0	0	0	0	16799.17733	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0	"https://gnps.ucsd.edu/ProteoSAFe/result.jsp?task=096df6e409a14363ab59cba2fd530e80&view=view_all_clusters_withID&show=true#{""main.cluster index_lowerinput"":""3.0"",""main.cluster index_upperinput"":""3.0""}"	https://gnps.ucsd.edu/ProteoSAFe/result.jsp?view=network_displayer&componentindex=10&task=096df6e409a14363ab59cba2fd530e80&show=true	N/A	"NCGC00380522-01_C20H30O4_1-Phenanthrenecarboxylic acid, 1,2,3,4,4a,4b,5,6,7,9,10,10a-dodecahydro-7-hydroxy-1,4a-dimethyl-7-(1-methylethyl)-9-oxo-"	0.982019	 	0.3455	0.3455	0	CC(C)C\1(O)CCC2\C(=C1)C(=O)CC3C2(C)CCCC3(C)C(O)=O	CCMSLIB00003139769	6627693.442	4	2	10	 	4	685.24	1	685.24	6627693.442