add na_rep="n/a" to all to_csv

Author: tien-tong

cubids/cubids.py

@@ -1449,9 +1449,9 @@ def get_tsvs(self, path_prefix):
         with open(summary_json, "w") as outfile:
             json.dump(summary_dict, outfile, indent=4)
 
-        big_df.to_csv(files_tsv, sep="\t", index=False)
+        big_df.to_csv(files_tsv, sep="\t", index=False, na_rep="n/a")
 
-        summary.to_csv(summary_tsv, sep="\t", index=False)
+        summary.to_csv(summary_tsv, sep="\t", index=False, na_rep="n/a")
 
         # Calculate the acq groups
         group_by_acquisition_sets(files_tsv, path_prefix, self.acq_group_level)

cubids/metadata_merge.py

@@ -381,7 +381,7 @@ def group_by_acquisition_sets(files_tsv, output_prefix, acq_group_level, is_long
 
     # Write the mapping of subject/session to
     acq_group_df = pd.DataFrame(grouped_sub_sess)
-    acq_group_df.to_csv(output_prefix + "_AcqGrouping.tsv", sep="\t", index=False)
+    acq_group_df.to_csv(output_prefix + "_AcqGrouping.tsv", sep="\t", index=False, na_rep="n/a")
 
     # Create data dictionary for acq group tsv
     acq_dict = get_acq_dictionary(is_longitudinal)

cubids/tests/test_apply.py

@@ -323,12 +323,12 @@ def test_cubids_apply_intendedfor(
     # Create a CuBIDS summary tsv
     summary_tsv = tmpdir / "summary.tsv"
     df = pd.DataFrame(summary_data)
-    df.to_csv(summary_tsv, sep="\t", index=False)
+    df.to_csv(summary_tsv, sep="\t", index=False, na_rep="n/a")
 
     # Create a CuBIDS files tsv
     files_tsv = tmpdir / "files.tsv"
     df = pd.DataFrame(fdata)
-    df.to_csv(files_tsv, sep="\t", index=False)
+    df.to_csv(files_tsv, sep="\t", index=False, na_rep="n/a")
 
     # Run cubids apply
     if isinstance(expected, str):

cubids/tests/test_bond.py

@@ -459,7 +459,7 @@ def test_tsv_merge_no_datalad(tmp_path):
     summary_df.loc[fa_nan_dwi_row, "MergeInto"] = summary_df.ParamGroup[complete_dwi_row]
 
     valid_tsv_file = tsv_prefix + "_valid_summary.tsv"
-    summary_df.to_csv(valid_tsv_file, sep="\t", index=False)
+    summary_df.to_csv(valid_tsv_file, sep="\t", index=False, na_rep="n/a")
 
     # about to apply merges!
 
@@ -472,7 +472,7 @@ def test_tsv_merge_no_datalad(tmp_path):
         complete_dwi_row
     ]
     invalid_tsv_file = tsv_prefix + "_invalid_summary.tsv"
-    summary_df.to_csv(invalid_tsv_file, sep="\t", index=False)
+    summary_df.to_csv(invalid_tsv_file, sep="\t", index=False, na_rep="n/a")
 
     with pytest.raises(Exception):
         bod.apply_tsv_changes(
@@ -572,7 +572,7 @@ def test_tsv_merge_changes(tmp_path):
     summary_df.loc[fa_nan_dwi_row, "MergeInto"] = summary_df.ParamGroup[complete_dwi_row]
 
     valid_tsv_file = tsv_prefix + "_valid_summary.tsv"
-    summary_df.to_csv(valid_tsv_file, sep="\t", index=False)
+    summary_df.to_csv(valid_tsv_file, sep="\t", index=False, na_rep="n/a")
 
     # about to merge
     bod.apply_tsv_changes(valid_tsv_file, original_files_tsv, str(tmp_path / "ok_modified"))
@@ -584,7 +584,7 @@ def test_tsv_merge_changes(tmp_path):
         complete_dwi_row
     ]
     invalid_tsv_file = tsv_prefix + "_invalid_summary.tsv"
-    summary_df.to_csv(invalid_tsv_file, sep="\t", index=False)
+    summary_df.to_csv(invalid_tsv_file, sep="\t", index=False, na_rep="n/a")
 
     with pytest.raises(Exception):
         bod.apply_tsv_changes(

cubids/tests/utils.py

@@ -108,7 +108,7 @@ def _add_deletion(summary_tsv):
     """
     df = pd.read_table(summary_tsv)
     df.loc[3, "MergeInto"] = 0
-    df.to_csv(summary_tsv, sep="\t", index=False)
+    df.to_csv(summary_tsv, sep="\t", index=False, na_rep="n/a")
     return df.loc[3, "KeyParamGroup"]
 
 

cubids/workflows.py

@@ -176,10 +176,13 @@ def _link_or_copy(src_path, dst_path):
                         participants_tsv_path,
                         sep="\t",
                         index=False,
+                        na_rep="n/a",
                     )
-            except Exception as e:  # noqa: F841
-                # Non-fatal: continue validation even if filtering fails
-                pass
+            except Exception as e:
+                logger.warning(
+                    f"Failed to filter participants.tsv for subject {subject}: {e}. "
+                    "Continuing validation without filtering."
+                )
 
         # Run the validator
         call = build_validator_call(
@@ -281,7 +284,7 @@ def validate(
             else:
                 val_tsv = str(bids_dir) + "/code/CuBIDS/" + str(output_prefix) + "_validation.tsv"
 
-            parsed.to_csv(val_tsv, sep="\t", index=False)
+            parsed.to_csv(val_tsv, sep="\t", index=False, na_rep="n/a")
 
             # build validation data dictionary json sidecar
             val_dict = get_val_dictionary()
@@ -373,7 +376,7 @@ def validate(
             else:
                 val_tsv = str(bids_dir) + "/code/CuBIDS/" + str(output_prefix) + "_validation.tsv"
 
-            parsed.to_csv(val_tsv, sep="\t", index=False)
+            parsed.to_csv(val_tsv, sep="\t", index=False, na_rep="n/a")
 
             # build validation data dictionary json sidecar
             val_dict = get_val_dictionary()