55import json
66import logging
77import multiprocessing as mp
8+ import ntpath
89import os
910import os .path as op
1011import re
@@ -543,11 +544,44 @@ def _validate_images_df(image_df):
543544 image_df : :class:`pandas.DataFrame`
544545 DataFrame with updated paths and columns.
545546 """
547+ image_df = image_df .copy (deep = False )
548+
546549 valid_suffixes = [".brik" , ".head" , ".nii" , ".img" , ".hed" ]
547550 id_columns = set (["id" , "study_id" , "contrast_id" ])
551+
552+ if image_df .columns .has_duplicates :
553+ merged_columns = {}
554+ for col in dict .fromkeys (image_df .columns ):
555+ values = image_df .loc [:, col ]
556+ if isinstance (values , pd .DataFrame ):
557+ merged_columns [col ] = values .bfill (axis = 1 ).iloc [:, 0 ]
558+ else :
559+ merged_columns [col ] = values
560+ image_df = pd .DataFrame (merged_columns , index = image_df .index )
561+
562+ def _is_absolute_path (value ):
563+ if not isinstance (value , str ):
564+ return False
565+ if not value :
566+ return False
567+ if value [0 ] == "/" :
568+ return True
569+ if value .startswith ("\\ \\ " ) or value .startswith ("//" ):
570+ return True
571+ return len (value ) > 2 and value [1 ] == ":" and value [2 ] in ("\\ " , "/" )
572+
573+ def _path_module_for (value ):
574+ if len (value ) > 1 and value [1 ] == ":" :
575+ return ntpath
576+ if value .startswith ("\\ " ):
577+ return ntpath
578+ return op
579+
548580 # Find columns in the DataFrame with images
549581 file_cols = []
550- for col in set (image_df .columns ) - id_columns :
582+ for col in image_df .columns :
583+ if col in id_columns :
584+ continue
551585 vals = [v for v in image_df [col ].values if isinstance (v , str )]
552586 fc = any ([any ([vs in v for vs in valid_suffixes ]) for v in vals ])
553587 if fc :
@@ -558,7 +592,7 @@ def _validate_images_df(image_df):
558592 abs_cols = []
559593 for col in file_cols :
560594 files = image_df [col ].tolist ()
561- abspaths = [f == op . abspath (f ) for f in files if isinstance (f , str )]
595+ abspaths = [_is_absolute_path (f ) for f in files if isinstance (f , str )]
562596 if all (abspaths ):
563597 abs_cols .append (col )
564598 elif not any (abspaths ):
@@ -570,28 +604,52 @@ def _validate_images_df(image_df):
570604 )
571605
572606 # Set relative paths from absolute ones
573- if len (abs_cols ):
574- all_files = list (np .ravel (image_df [abs_cols ].values ))
575- all_files = [f for f in all_files if isinstance (f , str )]
607+ for abs_col in abs_cols :
608+ rel_col = abs_col + "__relative"
609+ abs_values = image_df [abs_col ].tolist ()
610+
611+ if rel_col in image_df .columns :
612+ rel_values = image_df [rel_col ].tolist ()
613+ missing_relative = any (
614+ isinstance (abs_val , str ) and (not isinstance (rel_val , str ) or not rel_val )
615+ for abs_val , rel_val in zip (abs_values , rel_values )
616+ )
617+ if not missing_relative :
618+ continue
619+ else :
620+ rel_values = [None ] * len (abs_values )
621+
622+ string_files = [f for f in abs_values if isinstance (f , str )]
623+ if not string_files :
624+ continue
576625
577- if len (all_files ) == 1 :
578- # In the odd case where there's only one absolute path
579- shared_path = op .dirname (all_files [0 ]) + op .sep
626+ pathmod = _path_module_for (string_files [0 ])
627+ normalized_files = [pathmod .normpath (f ) if isinstance (f , str ) else f for f in abs_values ]
628+ string_dirs = [pathmod .dirname (f ) for f in normalized_files if isinstance (f , str )]
629+
630+ if len (string_dirs ) == 1 :
631+ shared_path = string_dirs [0 ].rstrip (pathmod .sep ) + pathmod .sep
580632 else :
581- shared_path = _find_stem (all_files )
633+ shared_path = pathmod .commonprefix (string_dirs )
634+ if not shared_path .endswith (pathmod .sep ):
635+ shared_path = pathmod .dirname (shared_path )
636+ shared_path = shared_path .rstrip (pathmod .sep ) + pathmod .sep
582637
583- # Get parent *directory* if shared path includes common prefix.
584- if not shared_path .endswith (op .sep ):
585- shared_path = op .dirname (shared_path ) + op .sep
586638 LGR .info (f"Shared path detected: '{ shared_path } )
587639
588- image_df_out = image_df .copy () # To avoid SettingWithCopyWarning
589- for abs_col in abs_cols :
590- image_df_out [abs_col + "__relative" ] = image_df [abs_col ].apply (
591- lambda x : x .split (shared_path )[1 ] if isinstance (x , str ) else x
592- )
640+ relative_values = []
641+ for norm_value , rel_value in zip (normalized_files , rel_values ):
642+ if isinstance (rel_value , str ) and rel_value :
643+ relative_values .append (rel_value )
644+ elif isinstance (norm_value , str ):
645+ if shared_path and norm_value .startswith (shared_path ):
646+ relative_values .append (norm_value [len (shared_path ) :])
647+ else :
648+ relative_values .append (pathmod .basename (norm_value ))
649+ else :
650+ relative_values .append (rel_value )
593651
594- image_df = image_df_out
652+ image_df [ rel_col ] = relative_values
595653
596654 # Normalize missing values to None (avoid NaN floats in path columns).
597655 # Pandas may keep float dtypes; force object to retain None.
@@ -729,7 +787,8 @@ def memmap_context(self, *args, **kwargs):
729787 self .memmap_filenames , filenames = [], []
730788 for i_file in range (n_files ):
731789 start_time = datetime .datetime .now ().strftime ("%Y%m%dT%H%M%S" )
732- _ , filename = mkstemp (prefix = self .__class__ .__name__ , suffix = start_time )
790+ fd , filename = mkstemp (prefix = self .__class__ .__name__ , suffix = start_time )
791+ os .close (fd )
733792 logger .debug (f"Temporary file written to { filename } )
734793 self .memmap_filenames .append (filename )
735794 filenames .append (filename )
0 commit comments