Require unique values for join column in secondary file to be joined (#471)

Author: alecw

src/python/dropseq_metadata/src/dropseq_metadata/join_and_filter_tsv.py

@@ -40,6 +40,11 @@
 import argparse
 import sys
 import pandas as pd
+from pandas.errors import MergeError
+try:
+    from . import cli
+except ImportError:
+    import cli
 
 DELETEME_COLUMN_SUFFIX = '_deleteme'
 
@@ -100,8 +105,13 @@ def main(options):
     for join_file, input_col, join_col in options.join:
         join_col_in_left = join_col in primary.columns
         secondary = pd.read_csv(join_file, sep='\t')
-        primary = primary.merge(secondary, how='left', left_on=input_col, right_on=join_col,
-                                suffixes=(None, DELETEME_COLUMN_SUFFIX))
+        try:
+            # many_to_one: require that the join column in the secondary file is unique
+            primary = primary.merge(secondary, how='left', left_on=input_col, right_on=join_col, validate="many_to_one",
+                                    suffixes=(None, DELETEME_COLUMN_SUFFIX))
+        except MergeError as e:
+            cli.logger.error(f"Error joining {join_file} on {input_col} and {join_col}: {e}")
+            return 1
         if not join_col_in_left:
             # drop the join column from the merged data frame
             primary.drop(join_col, axis=1, inplace=True)

src/python/dropseq_metadata/tests/test_join_and_filter_tsv.py

@@ -157,6 +157,13 @@ def test_include_exclude(self):
         self.assertTrue((outputDf["DONOR"].isin(donorsToInclude)).all())
         self.assertFalse((outputDf["predClass"].isin(predClassesToExclude)).any())
 
+    def test_negative_non_unique_join(self):
+        primary = os.path.join(self.testDataDir, "sample1.100.cell_metadata.txt")
+        secondary = os.path.join(self.testDataDir, "sample1.nonunique.scPred.txt")
+        options = self.options._replace(input=open(primary),
+                                        join=[(secondary, "CELL_BARCODE", "CELL_BARCODE"),
+                                              (secondary, "CELL_BARCODE", "CELL_BARCODE")])
+        self.assertEqual(dropseq_metadata.join_and_filter_tsv.main(options), 1)
 
     def assertSharedColumnsEqual(self, wideFile, narrowFile, wideRows = None, narrowRows = None, dropColumns = None):
         wideDf = pd.read_csv(wideFile, sep='\t', index_col=False)

testdata/python/dropseq_metadata/join_and_filter_tsv/sample1.nonunique.scPred.txt

@@ -0,0 +1,3 @@
+CELL_BARCODE	predClass	max.prob	predClass_allen_gluta_subclass	max.prob_allen_gluta_subclass	predClass_allen_gluta_subtype	max.prob_allen_gluta_subtype	predClass_allen_gaba_subclass	max.prob_allen_gaba_subclass	predClass_allen_gaba_subtype	max.prob_allen_gaba_subtype	doublet
+TGACTTTGTACGCTGC	glutamatergic	0.7	L23IT	1	Exc L2-3 LINC00507 FREM3	0.78	NA	NA	NA	NA	FALSE
+TGACTTTGTACGCTGC	glutamatergic	1	L23IT	1	Exc L2-3 LINC00507 FREM3	0.667	NA	NA	NA	NA	FALSE