imported copy_tree from master

Author: andrewprzh

assembler/src/spades_pipeline/stages/correction_stage.py

@@ -10,7 +10,6 @@
 import os
 import shutil
 import sys
-from distutils import dir_util
 from site import addsitedir
 
 from stages import stage
@@ -71,7 +70,7 @@ def generate_config(self, cfg):
         dst_configs = os.path.join(self.cfg.output_dir, "configs")
         if os.path.isdir(dst_configs):
             shutil.rmtree(dst_configs)
-        dir_util.copy_tree(os.path.join(self.tmp_configs_dir, "corrector"), dst_configs, preserve_times=False)
+        support.copy_tree(os.path.join(self.tmp_configs_dir, "corrector"), dst_configs, preserve_times=False)
         cfg_file_name = os.path.join(dst_configs, "corrector.info")
 
         self.cfg.tmp_dir = support.get_tmp_dir(prefix="corrector_")

assembler/src/spades_pipeline/stages/error_correction_stage.py

@@ -10,7 +10,6 @@
 import os
 import shutil
 import sys
-from distutils import dir_util
 from site import addsitedir
 
 import commands_parser
@@ -63,10 +62,10 @@ def generate_config(self, cfg):
         if os.path.isdir(dst_configs):
             shutil.rmtree(dst_configs)
         if cfg.iontorrent:
-            dir_util.copy_tree(os.path.join(self.tmp_configs_dir, "ionhammer"), dst_configs, preserve_times=False)
+            support.copy_tree(os.path.join(self.tmp_configs_dir, "ionhammer"), dst_configs, preserve_times=False)
             cfg_file_name = os.path.join(dst_configs, "ionhammer.cfg")
         else:
-            dir_util.copy_tree(os.path.join(self.tmp_configs_dir, "hammer"), dst_configs, preserve_times=False)
+            support.copy_tree(os.path.join(self.tmp_configs_dir, "hammer"), dst_configs, preserve_times=False)
             cfg_file_name = os.path.join(dst_configs, "config.info")
 
         cfg.tmp_dir = support.get_tmp_dir(prefix="hammer_")

assembler/src/spades_pipeline/stages/pipeline.py

@@ -9,8 +9,8 @@
 
 import os
 import shutil
-from distutils import dir_util
 
+import support
 import options_storage
 
 class Pipeline(object):
@@ -22,10 +22,10 @@ def copy_configs(self, cfg, spades_home, tmp_configs_dir):
             shutil.rmtree(tmp_configs_dir)
         if not os.path.isdir(tmp_configs_dir):
             if options_storage.args.configs_dir:
-                dir_util.copy_tree(options_storage.args.configs_dir, tmp_configs_dir, preserve_times=False,
+                support.copy_tree(options_storage.args.configs_dir, tmp_configs_dir, preserve_times=False,
                                    preserve_mode=False)
             else:
-                dir_util.copy_tree(os.path.join(spades_home, "configs"), tmp_configs_dir, preserve_times=False,
+                support.copy_tree(os.path.join(spades_home, "configs"), tmp_configs_dir, preserve_times=False,
                                    preserve_mode=False)
 
     def add(self, stage):

assembler/src/spades_pipeline/stages/spades_iteration_stage.py

@@ -9,11 +9,11 @@
 
 import os
 import shutil
-from distutils import dir_util
 
 import commands_parser
 import options_storage
 from stages import stage
+import support
 import process_cfg
 from process_cfg import bool_to_str
 
@@ -139,9 +139,7 @@ def generate_config(self, cfg):
         if self.get_stage(self.short_name) == options_storage.BASE_STAGE:
             if not os.path.isdir(data_dir):
                 os.makedirs(data_dir)
-
-            dir_util._path_created = {}  # see http://stackoverflow.com/questions/9160227/dir-util-copy-tree-fails-after-shutil-rmtree
-            dir_util.copy_tree(os.path.join(self.tmp_configs_dir, "debruijn"), dst_configs, preserve_times=False)
+            support.copy_tree(os.path.join(self.tmp_configs_dir, "debruijn"), dst_configs, preserve_times=False)
 
         if self.prev_K:
             additional_contigs_dname = os.path.join(cfg.output_dir, "K%d" % self.prev_K, "simplified_contigs")

assembler/src/spades_pipeline/support.py

@@ -940,3 +940,30 @@ def is_int(value):
         return True
     except ValueError:
         return False
+
+
+# shutil.copyfile does not copy any metadata (time and permission), so one
+# cannot expect preserve_mode = False and preserve_times = True to work.
+def copy_tree(src, dst, preserve_times=True, preserve_mode=True):
+    if sys.version.split()[0][0] == '2':
+        from distutils import dir_util
+        dir_util._path_created = {}  # see http://stackoverflow.com/questions/9160227/dir-util-copy-tree-fails-after-shutil-rmtree
+        dir_util.copy_tree(src, dst, preserve_times=preserve_times, preserve_mode=preserve_mode)
+        return
+
+    if not preserve_mode:
+        copy_fn = shutil.copyfile
+    else:
+        copy_fn = shutil.copy2
+
+    if os.path.exists(dst):
+        shutil.rmtree(dst)
+
+    # shutil.copytree preserves the timestamp, so we must update it afterwards.
+    shutil.copytree(src, dst, copy_function = copy_fn)
+
+    if not preserve_times:
+        for dirpath, _, filenames in os.walk(dst):
+            os.utime(dirpath, None)
+            for file in filenames:
+                os.utime(os.path.join(dirpath, file), None)